| { |
| "best_metric": 1.3996269702911377, |
| "best_model_checkpoint": "saves/Gemma-2B-It/lora/orpo/checkpoint-1500", |
| "epoch": 2.997999555456768, |
| "eval_steps": 500, |
| "global_step": 1686, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.017781729273171815, |
| "grad_norm": 1.9148550033569336, |
| "learning_rate": 4.9995745934141085e-06, |
| "logits/chosen": -22.071788787841797, |
| "logits/rejected": -21.994897842407227, |
| "logps/chosen": -1.9321304559707642, |
| "logps/rejected": -2.141274929046631, |
| "loss": 2.0148, |
| "odds_ratio_loss": 0.8263328671455383, |
| "rewards/accuracies": 0.42500001192092896, |
| "rewards/chosen": -0.19321303069591522, |
| "rewards/margins": 0.020914455875754356, |
| "rewards/rejected": -0.21412746608257294, |
| "sft_loss": 1.9321304559707642, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.03556345854634363, |
| "grad_norm": 4.772641181945801, |
| "learning_rate": 4.9982812903243405e-06, |
| "logits/chosen": -21.850475311279297, |
| "logits/rejected": -22.140661239624023, |
| "logps/chosen": -2.000199556350708, |
| "logps/rejected": -1.9620949029922485, |
| "loss": 2.0912, |
| "odds_ratio_loss": 0.9096724390983582, |
| "rewards/accuracies": 0.4437499940395355, |
| "rewards/chosen": -0.2000199258327484, |
| "rewards/margins": -0.003810454858466983, |
| "rewards/rejected": -0.19620949029922485, |
| "sft_loss": 2.000199556350708, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.05334518781951545, |
| "grad_norm": 1.9645005464553833, |
| "learning_rate": 4.996120496405222e-06, |
| "logits/chosen": -22.181926727294922, |
| "logits/rejected": -22.236988067626953, |
| "logps/chosen": -1.9057893753051758, |
| "logps/rejected": -2.2623982429504395, |
| "loss": 1.9768, |
| "odds_ratio_loss": 0.7102858424186707, |
| "rewards/accuracies": 0.550000011920929, |
| "rewards/chosen": -0.19057895243167877, |
| "rewards/margins": 0.03566090017557144, |
| "rewards/rejected": -0.22623984515666962, |
| "sft_loss": 1.9057893753051758, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.07112691709268726, |
| "grad_norm": 1.9976199865341187, |
| "learning_rate": 4.99309296196014e-06, |
| "logits/chosen": -22.178194046020508, |
| "logits/rejected": -22.227825164794922, |
| "logps/chosen": -1.8588358163833618, |
| "logps/rejected": -2.0477230548858643, |
| "loss": 1.94, |
| "odds_ratio_loss": 0.8119063377380371, |
| "rewards/accuracies": 0.512499988079071, |
| "rewards/chosen": -0.18588361144065857, |
| "rewards/margins": 0.01888870820403099, |
| "rewards/rejected": -0.20477227866649628, |
| "sft_loss": 1.8588358163833618, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.08890864636585907, |
| "grad_norm": 1.6904418468475342, |
| "learning_rate": 4.989199738255166e-06, |
| "logits/chosen": -22.063446044921875, |
| "logits/rejected": -22.088878631591797, |
| "logps/chosen": -1.8785845041275024, |
| "logps/rejected": -2.0510427951812744, |
| "loss": 1.9601, |
| "odds_ratio_loss": 0.81475830078125, |
| "rewards/accuracies": 0.512499988079071, |
| "rewards/chosen": -0.187858447432518, |
| "rewards/margins": 0.0172458253800869, |
| "rewards/rejected": -0.2051042765378952, |
| "sft_loss": 1.8785845041275024, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.1066903756390309, |
| "grad_norm": 1.597947359085083, |
| "learning_rate": 4.984442177154031e-06, |
| "logits/chosen": -22.34821319580078, |
| "logits/rejected": -22.315746307373047, |
| "logps/chosen": -1.9788051843643188, |
| "logps/rejected": -2.084188461303711, |
| "loss": 2.0713, |
| "odds_ratio_loss": 0.924887478351593, |
| "rewards/accuracies": 0.5562499761581421, |
| "rewards/chosen": -0.19788053631782532, |
| "rewards/margins": 0.01053832471370697, |
| "rewards/rejected": -0.20841887593269348, |
| "sft_loss": 1.9788051843643188, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.12447210491220272, |
| "grad_norm": 2.274142026901245, |
| "learning_rate": 4.978821930648704e-06, |
| "logits/chosen": -22.288013458251953, |
| "logits/rejected": -22.197546005249023, |
| "logps/chosen": -1.9143011569976807, |
| "logps/rejected": -1.8838450908660889, |
| "loss": 2.0168, |
| "odds_ratio_loss": 1.0252134799957275, |
| "rewards/accuracies": 0.4625000059604645, |
| "rewards/chosen": -0.19143010675907135, |
| "rewards/margins": -0.0030455999076366425, |
| "rewards/rejected": -0.1883845031261444, |
| "sft_loss": 1.9143011569976807, |
| "step": 70 |
| }, |
| { |
| "epoch": 0.14225383418537452, |
| "grad_norm": 2.63519549369812, |
| "learning_rate": 4.97234095028576e-06, |
| "logits/chosen": -22.663928985595703, |
| "logits/rejected": -22.51036834716797, |
| "logps/chosen": -1.8908298015594482, |
| "logps/rejected": -1.9426231384277344, |
| "loss": 1.9749, |
| "odds_ratio_loss": 0.8411667943000793, |
| "rewards/accuracies": 0.4937500059604645, |
| "rewards/chosen": -0.1890830099582672, |
| "rewards/margins": 0.005179307423532009, |
| "rewards/rejected": -0.1942623108625412, |
| "sft_loss": 1.8908298015594482, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.16003556345854633, |
| "grad_norm": 1.5385671854019165, |
| "learning_rate": 4.965001486488743e-06, |
| "logits/chosen": -22.35540199279785, |
| "logits/rejected": -22.453685760498047, |
| "logps/chosen": -1.6930122375488281, |
| "logps/rejected": -1.8899803161621094, |
| "loss": 1.7643, |
| "odds_ratio_loss": 0.7125129699707031, |
| "rewards/accuracies": 0.5375000238418579, |
| "rewards/chosen": -0.16930122673511505, |
| "rewards/margins": 0.019696824252605438, |
| "rewards/rejected": -0.1889980286359787, |
| "sft_loss": 1.6930122375488281, |
| "step": 90 |
| }, |
| { |
| "epoch": 0.17781729273171815, |
| "grad_norm": 1.6486105918884277, |
| "learning_rate": 4.956806087776732e-06, |
| "logits/chosen": -22.912220001220703, |
| "logits/rejected": -22.764265060424805, |
| "logps/chosen": -1.7519505023956299, |
| "logps/rejected": -2.004110813140869, |
| "loss": 1.8222, |
| "odds_ratio_loss": 0.7028593420982361, |
| "rewards/accuracies": 0.550000011920929, |
| "rewards/chosen": -0.17519506812095642, |
| "rewards/margins": 0.025216031819581985, |
| "rewards/rejected": -0.2004111111164093, |
| "sft_loss": 1.7519505023956299, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.19559902200489, |
| "grad_norm": 2.1504974365234375, |
| "learning_rate": 4.947757599879411e-06, |
| "logits/chosen": -22.865467071533203, |
| "logits/rejected": -23.005489349365234, |
| "logps/chosen": -1.727837324142456, |
| "logps/rejected": -1.8906141519546509, |
| "loss": 1.8026, |
| "odds_ratio_loss": 0.7475000023841858, |
| "rewards/accuracies": 0.518750011920929, |
| "rewards/chosen": -0.1727837324142456, |
| "rewards/margins": 0.016277695074677467, |
| "rewards/rejected": -0.18906141817569733, |
| "sft_loss": 1.727837324142456, |
| "step": 110 |
| }, |
| { |
| "epoch": 0.2133807512780618, |
| "grad_norm": 3.6934735774993896, |
| "learning_rate": 4.937859164748931e-06, |
| "logits/chosen": -22.859783172607422, |
| "logits/rejected": -23.031169891357422, |
| "logps/chosen": -1.5483535528182983, |
| "logps/rejected": -1.6135647296905518, |
| "loss": 1.624, |
| "odds_ratio_loss": 0.7560455203056335, |
| "rewards/accuracies": 0.5062500238418579, |
| "rewards/chosen": -0.15483535826206207, |
| "rewards/margins": 0.006521114148199558, |
| "rewards/rejected": -0.16135647892951965, |
| "sft_loss": 1.5483535528182983, |
| "step": 120 |
| }, |
| { |
| "epoch": 0.23116248055123362, |
| "grad_norm": 3.72802734375, |
| "learning_rate": 4.92711421946891e-06, |
| "logits/chosen": -23.100276947021484, |
| "logits/rejected": -22.69415283203125, |
| "logps/chosen": -1.5568244457244873, |
| "logps/rejected": -1.8098100423812866, |
| "loss": 1.6294, |
| "odds_ratio_loss": 0.7258477210998535, |
| "rewards/accuracies": 0.5062500238418579, |
| "rewards/chosen": -0.15568244457244873, |
| "rewards/margins": 0.025298580527305603, |
| "rewards/rejected": -0.18098104000091553, |
| "sft_loss": 1.5568244457244873, |
| "step": 130 |
| }, |
| { |
| "epoch": 0.24894420982440543, |
| "grad_norm": 5.469463348388672, |
| "learning_rate": 4.915526495060961e-06, |
| "logits/chosen": -23.371618270874023, |
| "logits/rejected": -23.214031219482422, |
| "logps/chosen": -1.4800597429275513, |
| "logps/rejected": -1.746651291847229, |
| "loss": 1.5507, |
| "odds_ratio_loss": 0.7065833806991577, |
| "rewards/accuracies": 0.574999988079071, |
| "rewards/chosen": -0.14800596237182617, |
| "rewards/margins": 0.026659172028303146, |
| "rewards/rejected": -0.17466513812541962, |
| "sft_loss": 1.4800597429275513, |
| "step": 140 |
| }, |
| { |
| "epoch": 0.26672593909757725, |
| "grad_norm": 4.669017314910889, |
| "learning_rate": 4.903100015189153e-06, |
| "logits/chosen": -22.959320068359375, |
| "logits/rejected": -23.156007766723633, |
| "logps/chosen": -1.5119131803512573, |
| "logps/rejected": -1.708356261253357, |
| "loss": 1.5853, |
| "odds_ratio_loss": 0.7340201735496521, |
| "rewards/accuracies": 0.5062500238418579, |
| "rewards/chosen": -0.1511913239955902, |
| "rewards/margins": 0.01964429020881653, |
| "rewards/rejected": -0.17083561420440674, |
| "sft_loss": 1.5119131803512573, |
| "step": 150 |
| }, |
| { |
| "epoch": 0.28450766837074903, |
| "grad_norm": 1.5560555458068848, |
| "learning_rate": 4.889839094762848e-06, |
| "logits/chosen": -22.915985107421875, |
| "logits/rejected": -22.794408798217773, |
| "logps/chosen": -1.6158710718154907, |
| "logps/rejected": -1.7288596630096436, |
| "loss": 1.6909, |
| "odds_ratio_loss": 0.7498828768730164, |
| "rewards/accuracies": 0.5249999761581421, |
| "rewards/chosen": -0.16158713400363922, |
| "rewards/margins": 0.011298839002847672, |
| "rewards/rejected": -0.1728859841823578, |
| "sft_loss": 1.6158710718154907, |
| "step": 160 |
| }, |
| { |
| "epoch": 0.3022893976439209, |
| "grad_norm": 1.565077304840088, |
| "learning_rate": 4.875748338438416e-06, |
| "logits/chosen": -23.140369415283203, |
| "logits/rejected": -23.20174789428711, |
| "logps/chosen": -1.532865285873413, |
| "logps/rejected": -1.6764837503433228, |
| "loss": 1.6069, |
| "odds_ratio_loss": 0.7403478026390076, |
| "rewards/accuracies": 0.48750001192092896, |
| "rewards/chosen": -0.15328654646873474, |
| "rewards/margins": 0.014361831359565258, |
| "rewards/rejected": -0.16764836013317108, |
| "sft_loss": 1.532865285873413, |
| "step": 170 |
| }, |
| { |
| "epoch": 0.32007112691709266, |
| "grad_norm": 3.263695240020752, |
| "learning_rate": 4.8608326390203386e-06, |
| "logits/chosen": -22.981613159179688, |
| "logits/rejected": -22.818286895751953, |
| "logps/chosen": -1.485670804977417, |
| "logps/rejected": -1.6812422275543213, |
| "loss": 1.5542, |
| "odds_ratio_loss": 0.6854217052459717, |
| "rewards/accuracies": 0.5375000238418579, |
| "rewards/chosen": -0.1485670804977417, |
| "rewards/margins": 0.019557146355509758, |
| "rewards/rejected": -0.1681242287158966, |
| "sft_loss": 1.485670804977417, |
| "step": 180 |
| }, |
| { |
| "epoch": 0.3378528561902645, |
| "grad_norm": 2.1444835662841797, |
| "learning_rate": 4.845097175762251e-06, |
| "logits/chosen": -23.199800491333008, |
| "logits/rejected": -23.2564640045166, |
| "logps/chosen": -1.4873155355453491, |
| "logps/rejected": -1.5498250722885132, |
| "loss": 1.5614, |
| "odds_ratio_loss": 0.7410100698471069, |
| "rewards/accuracies": 0.48124998807907104, |
| "rewards/chosen": -0.1487315446138382, |
| "rewards/margins": 0.006250949110835791, |
| "rewards/rejected": -0.15498249232769012, |
| "sft_loss": 1.4873155355453491, |
| "step": 190 |
| }, |
| { |
| "epoch": 0.3556345854634363, |
| "grad_norm": 5.516879558563232, |
| "learning_rate": 4.8285474125685286e-06, |
| "logits/chosen": -23.00992774963379, |
| "logits/rejected": -22.893043518066406, |
| "logps/chosen": -1.520996332168579, |
| "logps/rejected": -1.6076465845108032, |
| "loss": 1.5966, |
| "odds_ratio_loss": 0.7559183239936829, |
| "rewards/accuracies": 0.512499988079071, |
| "rewards/chosen": -0.1520996391773224, |
| "rewards/margins": 0.008664996363222599, |
| "rewards/rejected": -0.1607646495103836, |
| "sft_loss": 1.520996332168579, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.37341631473660813, |
| "grad_norm": 1.6477737426757812, |
| "learning_rate": 4.811189096097025e-06, |
| "logits/chosen": -22.601619720458984, |
| "logits/rejected": -22.704158782958984, |
| "logps/chosen": -1.5167438983917236, |
| "logps/rejected": -1.700338363647461, |
| "loss": 1.5896, |
| "odds_ratio_loss": 0.7285597920417786, |
| "rewards/accuracies": 0.5249999761581421, |
| "rewards/chosen": -0.15167437493801117, |
| "rewards/margins": 0.018359454348683357, |
| "rewards/rejected": -0.17003384232521057, |
| "sft_loss": 1.5167438983917236, |
| "step": 210 |
| }, |
| { |
| "epoch": 0.39119804400978, |
| "grad_norm": 2.6526737213134766, |
| "learning_rate": 4.793028253763633e-06, |
| "logits/chosen": -22.879850387573242, |
| "logits/rejected": -22.78567123413086, |
| "logps/chosen": -1.4604800939559937, |
| "logps/rejected": -1.6260970830917358, |
| "loss": 1.537, |
| "odds_ratio_loss": 0.7654477953910828, |
| "rewards/accuracies": 0.518750011920929, |
| "rewards/chosen": -0.14604800939559937, |
| "rewards/margins": 0.016561714932322502, |
| "rewards/rejected": -0.16260972619056702, |
| "sft_loss": 1.4604800939559937, |
| "step": 220 |
| }, |
| { |
| "epoch": 0.40897977328295176, |
| "grad_norm": 2.860865354537964, |
| "learning_rate": 4.774071191649352e-06, |
| "logits/chosen": -22.46622657775879, |
| "logits/rejected": -22.480607986450195, |
| "logps/chosen": -1.376908779144287, |
| "logps/rejected": -1.6316293478012085, |
| "loss": 1.4435, |
| "odds_ratio_loss": 0.6654683351516724, |
| "rewards/accuracies": 0.5687500238418579, |
| "rewards/chosen": -0.13769087195396423, |
| "rewards/margins": 0.025472048670053482, |
| "rewards/rejected": -0.16316291689872742, |
| "sft_loss": 1.376908779144287, |
| "step": 230 |
| }, |
| { |
| "epoch": 0.4267615025561236, |
| "grad_norm": 2.685337781906128, |
| "learning_rate": 4.7543244923105975e-06, |
| "logits/chosen": -22.682777404785156, |
| "logits/rejected": -22.806440353393555, |
| "logps/chosen": -1.5592434406280518, |
| "logps/rejected": -1.5708439350128174, |
| "loss": 1.6395, |
| "odds_ratio_loss": 0.8026041984558105, |
| "rewards/accuracies": 0.5062500238418579, |
| "rewards/chosen": -0.15592436492443085, |
| "rewards/margins": 0.0011600303696468472, |
| "rewards/rejected": -0.1570843905210495, |
| "sft_loss": 1.5592434406280518, |
| "step": 240 |
| }, |
| { |
| "epoch": 0.4445432318292954, |
| "grad_norm": 1.3707021474838257, |
| "learning_rate": 4.733795012493506e-06, |
| "logits/chosen": -22.8146915435791, |
| "logits/rejected": -22.913793563842773, |
| "logps/chosen": -1.5595623254776, |
| "logps/rejected": -1.6163349151611328, |
| "loss": 1.6357, |
| "odds_ratio_loss": 0.7609250545501709, |
| "rewards/accuracies": 0.4937500059604645, |
| "rewards/chosen": -0.1559562385082245, |
| "rewards/margins": 0.005677259061485529, |
| "rewards/rejected": -0.1616334766149521, |
| "sft_loss": 1.5595623254776, |
| "step": 250 |
| }, |
| { |
| "epoch": 0.46232496110246724, |
| "grad_norm": 1.3753399848937988, |
| "learning_rate": 4.712489880753035e-06, |
| "logits/chosen": -22.511287689208984, |
| "logits/rejected": -22.446317672729492, |
| "logps/chosen": -1.3289070129394531, |
| "logps/rejected": -1.4945290088653564, |
| "loss": 1.3952, |
| "odds_ratio_loss": 0.6626302003860474, |
| "rewards/accuracies": 0.5562499761581421, |
| "rewards/chosen": -0.1328907012939453, |
| "rewards/margins": 0.016562188044190407, |
| "rewards/rejected": -0.14945289492607117, |
| "sft_loss": 1.3289070129394531, |
| "step": 260 |
| }, |
| { |
| "epoch": 0.480106690375639, |
| "grad_norm": 5.678652286529541, |
| "learning_rate": 4.690416494977673e-06, |
| "logits/chosen": -22.829517364501953, |
| "logits/rejected": -22.87631607055664, |
| "logps/chosen": -1.4606059789657593, |
| "logps/rejected": -1.6754430532455444, |
| "loss": 1.5279, |
| "odds_ratio_loss": 0.6730555295944214, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.14606061577796936, |
| "rewards/margins": 0.02148369327187538, |
| "rewards/rejected": -0.16754429042339325, |
| "sft_loss": 1.4606059789657593, |
| "step": 270 |
| }, |
| { |
| "epoch": 0.49788841964881086, |
| "grad_norm": 1.2074130773544312, |
| "learning_rate": 4.667582519820639e-06, |
| "logits/chosen": -22.504804611206055, |
| "logits/rejected": -22.659706115722656, |
| "logps/chosen": -1.479263424873352, |
| "logps/rejected": -1.5646381378173828, |
| "loss": 1.5534, |
| "odds_ratio_loss": 0.7413693070411682, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.14792636036872864, |
| "rewards/margins": 0.008537469431757927, |
| "rewards/rejected": -0.15646381676197052, |
| "sft_loss": 1.479263424873352, |
| "step": 280 |
| }, |
| { |
| "epoch": 0.5156701489219827, |
| "grad_norm": 2.7386221885681152, |
| "learning_rate": 4.643995884038443e-06, |
| "logits/chosen": -22.59560775756836, |
| "logits/rejected": -22.57559585571289, |
| "logps/chosen": -1.3870899677276611, |
| "logps/rejected": -1.5722427368164062, |
| "loss": 1.4562, |
| "odds_ratio_loss": 0.6912243962287903, |
| "rewards/accuracies": 0.5562499761581421, |
| "rewards/chosen": -0.13870900869369507, |
| "rewards/margins": 0.01851527765393257, |
| "rewards/rejected": -0.15722428262233734, |
| "sft_loss": 1.3870899677276611, |
| "step": 290 |
| }, |
| { |
| "epoch": 0.5334518781951545, |
| "grad_norm": 1.5327345132827759, |
| "learning_rate": 4.6196647777377475e-06, |
| "logits/chosen": -22.43231201171875, |
| "logits/rejected": -22.38507652282715, |
| "logps/chosen": -1.4321014881134033, |
| "logps/rejected": -1.4831379652023315, |
| "loss": 1.5072, |
| "odds_ratio_loss": 0.7509574294090271, |
| "rewards/accuracies": 0.4937500059604645, |
| "rewards/chosen": -0.14321014285087585, |
| "rewards/margins": 0.005103647243231535, |
| "rewards/rejected": -0.14831380546092987, |
| "sft_loss": 1.4321014881134033, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.5512336074683263, |
| "grad_norm": 1.276062250137329, |
| "learning_rate": 4.59459764953147e-06, |
| "logits/chosen": -22.478355407714844, |
| "logits/rejected": -22.29865264892578, |
| "logps/chosen": -1.445011854171753, |
| "logps/rejected": -1.550065279006958, |
| "loss": 1.5147, |
| "odds_ratio_loss": 0.6971566081047058, |
| "rewards/accuracies": 0.4625000059604645, |
| "rewards/chosen": -0.14450117945671082, |
| "rewards/margins": 0.010505353100597858, |
| "rewards/rejected": -0.1550065577030182, |
| "sft_loss": 1.445011854171753, |
| "step": 310 |
| }, |
| { |
| "epoch": 0.5690153367414981, |
| "grad_norm": 2.80613112449646, |
| "learning_rate": 4.568803203605133e-06, |
| "logits/chosen": -22.582855224609375, |
| "logits/rejected": -22.391347885131836, |
| "logps/chosen": -1.3941065073013306, |
| "logps/rejected": -1.580993413925171, |
| "loss": 1.4659, |
| "odds_ratio_loss": 0.7180419564247131, |
| "rewards/accuracies": 0.5874999761581421, |
| "rewards/chosen": -0.13941065967082977, |
| "rewards/margins": 0.018688684329390526, |
| "rewards/rejected": -0.15809933841228485, |
| "sft_loss": 1.3941065073013306, |
| "step": 320 |
| }, |
| { |
| "epoch": 0.58679706601467, |
| "grad_norm": 5.356297492980957, |
| "learning_rate": 4.542290396694462e-06, |
| "logits/chosen": -22.250286102294922, |
| "logits/rejected": -22.175914764404297, |
| "logps/chosen": -1.4387528896331787, |
| "logps/rejected": -1.5810470581054688, |
| "loss": 1.5106, |
| "odds_ratio_loss": 0.7184728980064392, |
| "rewards/accuracies": 0.512499988079071, |
| "rewards/chosen": -0.14387528598308563, |
| "rewards/margins": 0.014229406602680683, |
| "rewards/rejected": -0.15810470283031464, |
| "sft_loss": 1.4387528896331787, |
| "step": 330 |
| }, |
| { |
| "epoch": 0.6045787952878418, |
| "grad_norm": 8.996047019958496, |
| "learning_rate": 4.515068434975298e-06, |
| "logits/chosen": -22.00839614868164, |
| "logits/rejected": -22.072261810302734, |
| "logps/chosen": -1.4673653841018677, |
| "logps/rejected": -1.6608636379241943, |
| "loss": 1.5376, |
| "odds_ratio_loss": 0.7021427154541016, |
| "rewards/accuracies": 0.512499988079071, |
| "rewards/chosen": -0.14673654735088348, |
| "rewards/margins": 0.019349832087755203, |
| "rewards/rejected": -0.16608639061450958, |
| "sft_loss": 1.4673653841018677, |
| "step": 340 |
| }, |
| { |
| "epoch": 0.6223605245610135, |
| "grad_norm": 1.460726261138916, |
| "learning_rate": 4.487146770866887e-06, |
| "logits/chosen": -22.291297912597656, |
| "logits/rejected": -22.382854461669922, |
| "logps/chosen": -1.406706690788269, |
| "logps/rejected": -1.4625658988952637, |
| "loss": 1.4815, |
| "odds_ratio_loss": 0.747775673866272, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": -0.14067067205905914, |
| "rewards/margins": 0.005585917271673679, |
| "rewards/rejected": -0.14625659584999084, |
| "sft_loss": 1.406706690788269, |
| "step": 350 |
| }, |
| { |
| "epoch": 0.6401422538341853, |
| "grad_norm": 1.722812533378601, |
| "learning_rate": 4.458535099749666e-06, |
| "logits/chosen": -22.276639938354492, |
| "logits/rejected": -22.166675567626953, |
| "logps/chosen": -1.5117685794830322, |
| "logps/rejected": -1.5999605655670166, |
| "loss": 1.5911, |
| "odds_ratio_loss": 0.7935177087783813, |
| "rewards/accuracies": 0.518750011920929, |
| "rewards/chosen": -0.15117685496807098, |
| "rewards/margins": 0.008819197304546833, |
| "rewards/rejected": -0.15999604761600494, |
| "sft_loss": 1.5117685794830322, |
| "step": 360 |
| }, |
| { |
| "epoch": 0.6579239831073572, |
| "grad_norm": 2.568336248397827, |
| "learning_rate": 4.429243356598694e-06, |
| "logits/chosen": -21.958419799804688, |
| "logits/rejected": -21.927824020385742, |
| "logps/chosen": -1.4804319143295288, |
| "logps/rejected": -1.6579450368881226, |
| "loss": 1.5496, |
| "odds_ratio_loss": 0.6912356615066528, |
| "rewards/accuracies": 0.512499988079071, |
| "rewards/chosen": -0.1480431854724884, |
| "rewards/margins": 0.017751310020685196, |
| "rewards/rejected": -0.1657945215702057, |
| "sft_loss": 1.4804319143295288, |
| "step": 370 |
| }, |
| { |
| "epoch": 0.675705712380529, |
| "grad_norm": 1.4206441640853882, |
| "learning_rate": 4.399281712533875e-06, |
| "logits/chosen": -22.067081451416016, |
| "logits/rejected": -22.091421127319336, |
| "logps/chosen": -1.4124424457550049, |
| "logps/rejected": -1.4996305704116821, |
| "loss": 1.4873, |
| "odds_ratio_loss": 0.7487770318984985, |
| "rewards/accuracies": 0.4749999940395355, |
| "rewards/chosen": -0.14124426245689392, |
| "rewards/margins": 0.008718819357454777, |
| "rewards/rejected": -0.14996306598186493, |
| "sft_loss": 1.4124424457550049, |
| "step": 380 |
| }, |
| { |
| "epoch": 0.6934874416537008, |
| "grad_norm": 2.3953347206115723, |
| "learning_rate": 4.368660571288192e-06, |
| "logits/chosen": -22.193960189819336, |
| "logits/rejected": -22.225393295288086, |
| "logps/chosen": -1.394415020942688, |
| "logps/rejected": -1.500723123550415, |
| "loss": 1.4702, |
| "odds_ratio_loss": 0.7577108144760132, |
| "rewards/accuracies": 0.4749999940395355, |
| "rewards/chosen": -0.13944150507450104, |
| "rewards/margins": 0.010630805976688862, |
| "rewards/rejected": -0.15007230639457703, |
| "sft_loss": 1.394415020942688, |
| "step": 390 |
| }, |
| { |
| "epoch": 0.7112691709268726, |
| "grad_norm": 1.4220997095108032, |
| "learning_rate": 4.337390565595163e-06, |
| "logits/chosen": -21.68547248840332, |
| "logits/rejected": -21.761310577392578, |
| "logps/chosen": -1.464005708694458, |
| "logps/rejected": -1.5315691232681274, |
| "loss": 1.5392, |
| "odds_ratio_loss": 0.75159752368927, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.1464005708694458, |
| "rewards/margins": 0.006756337825208902, |
| "rewards/rejected": -0.15315690636634827, |
| "sft_loss": 1.464005708694458, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.7290509002000445, |
| "grad_norm": 1.8401212692260742, |
| "learning_rate": 4.305482553496786e-06, |
| "logits/chosen": -21.259353637695312, |
| "logits/rejected": -21.159082412719727, |
| "logps/chosen": -1.3970698118209839, |
| "logps/rejected": -1.5361021757125854, |
| "loss": 1.4689, |
| "odds_ratio_loss": 0.7183545827865601, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.13970699906349182, |
| "rewards/margins": 0.01390322856605053, |
| "rewards/rejected": -0.1536101996898651, |
| "sft_loss": 1.3970698118209839, |
| "step": 410 |
| }, |
| { |
| "epoch": 0.7468326294732163, |
| "grad_norm": 2.2964348793029785, |
| "learning_rate": 4.272947614573244e-06, |
| "logits/chosen": -21.679210662841797, |
| "logits/rejected": -21.884136199951172, |
| "logps/chosen": -1.447422981262207, |
| "logps/rejected": -1.5282857418060303, |
| "loss": 1.5205, |
| "odds_ratio_loss": 0.730276346206665, |
| "rewards/accuracies": 0.518750011920929, |
| "rewards/chosen": -0.14474229514598846, |
| "rewards/margins": 0.008086279034614563, |
| "rewards/rejected": -0.15282857418060303, |
| "sft_loss": 1.447422981262207, |
| "step": 420 |
| }, |
| { |
| "epoch": 0.7646143587463881, |
| "grad_norm": 1.2190438508987427, |
| "learning_rate": 4.23979704609569e-06, |
| "logits/chosen": -21.96237564086914, |
| "logits/rejected": -22.065784454345703, |
| "logps/chosen": -1.398108959197998, |
| "logps/rejected": -1.495884656906128, |
| "loss": 1.4676, |
| "odds_ratio_loss": 0.6946425437927246, |
| "rewards/accuracies": 0.543749988079071, |
| "rewards/chosen": -0.13981090486049652, |
| "rewards/margins": 0.009777549654245377, |
| "rewards/rejected": -0.1495884507894516, |
| "sft_loss": 1.398108959197998, |
| "step": 430 |
| }, |
| { |
| "epoch": 0.78239608801956, |
| "grad_norm": 2.3040215969085693, |
| "learning_rate": 4.206042359103435e-06, |
| "logits/chosen": -21.56629180908203, |
| "logits/rejected": -21.716127395629883, |
| "logps/chosen": -1.487396001815796, |
| "logps/rejected": -1.617078185081482, |
| "loss": 1.5602, |
| "odds_ratio_loss": 0.7281696796417236, |
| "rewards/accuracies": 0.543749988079071, |
| "rewards/chosen": -0.14873960614204407, |
| "rewards/margins": 0.012968212366104126, |
| "rewards/rejected": -0.1617078334093094, |
| "sft_loss": 1.487396001815796, |
| "step": 440 |
| }, |
| { |
| "epoch": 0.8001778172927317, |
| "grad_norm": 2.5727310180664062, |
| "learning_rate": 4.17169527440691e-06, |
| "logits/chosen": -21.884145736694336, |
| "logits/rejected": -21.738811492919922, |
| "logps/chosen": -1.4501018524169922, |
| "logps/rejected": -1.4668500423431396, |
| "loss": 1.5286, |
| "odds_ratio_loss": 0.7853611707687378, |
| "rewards/accuracies": 0.48124998807907104, |
| "rewards/chosen": -0.14501020312309265, |
| "rewards/margins": 0.0016747992485761642, |
| "rewards/rejected": -0.14668500423431396, |
| "sft_loss": 1.4501018524169922, |
| "step": 450 |
| }, |
| { |
| "epoch": 0.8179595465659035, |
| "grad_norm": 2.54972243309021, |
| "learning_rate": 4.136767718517797e-06, |
| "logits/chosen": -21.746496200561523, |
| "logits/rejected": -21.7362060546875, |
| "logps/chosen": -1.3023537397384644, |
| "logps/rejected": -1.5028297901153564, |
| "loss": 1.368, |
| "odds_ratio_loss": 0.6567283868789673, |
| "rewards/accuracies": 0.543749988079071, |
| "rewards/chosen": -0.13023535907268524, |
| "rewards/margins": 0.020047612488269806, |
| "rewards/rejected": -0.15028299391269684, |
| "sft_loss": 1.3023537397384644, |
| "step": 460 |
| }, |
| { |
| "epoch": 0.8357412758390753, |
| "grad_norm": 6.595831871032715, |
| "learning_rate": 4.1012718195077196e-06, |
| "logits/chosen": -21.96458625793457, |
| "logits/rejected": -22.172712326049805, |
| "logps/chosen": -1.4211918115615845, |
| "logps/rejected": -1.4663982391357422, |
| "loss": 1.4961, |
| "odds_ratio_loss": 0.7494389414787292, |
| "rewards/accuracies": 0.48750001192092896, |
| "rewards/chosen": -0.14211918413639069, |
| "rewards/margins": 0.004520639777183533, |
| "rewards/rejected": -0.14663982391357422, |
| "sft_loss": 1.4211918115615845, |
| "step": 470 |
| }, |
| { |
| "epoch": 0.8535230051122472, |
| "grad_norm": 3.3689377307891846, |
| "learning_rate": 4.065219902796953e-06, |
| "logits/chosen": -21.535301208496094, |
| "logits/rejected": -21.487293243408203, |
| "logps/chosen": -1.3686919212341309, |
| "logps/rejected": -1.5178884267807007, |
| "loss": 1.4414, |
| "odds_ratio_loss": 0.7275662422180176, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.13686920702457428, |
| "rewards/margins": 0.014919650740921497, |
| "rewards/rejected": -0.1517888456583023, |
| "sft_loss": 1.3686919212341309, |
| "step": 480 |
| }, |
| { |
| "epoch": 0.871304734385419, |
| "grad_norm": 1.1600650548934937, |
| "learning_rate": 4.028624486874608e-06, |
| "logits/chosen": -21.398052215576172, |
| "logits/rejected": -21.58942222595215, |
| "logps/chosen": -1.3973881006240845, |
| "logps/rejected": -1.5505540370941162, |
| "loss": 1.4689, |
| "odds_ratio_loss": 0.7148123383522034, |
| "rewards/accuracies": 0.5375000238418579, |
| "rewards/chosen": -0.1397387981414795, |
| "rewards/margins": 0.015316602773964405, |
| "rewards/rejected": -0.15505541861057281, |
| "sft_loss": 1.3973881006240845, |
| "step": 490 |
| }, |
| { |
| "epoch": 0.8890864636585908, |
| "grad_norm": 1.584820032119751, |
| "learning_rate": 3.99149827895177e-06, |
| "logits/chosen": -21.60881996154785, |
| "logits/rejected": -21.396835327148438, |
| "logps/chosen": -1.4577990770339966, |
| "logps/rejected": -1.5503555536270142, |
| "loss": 1.5296, |
| "odds_ratio_loss": 0.7181479334831238, |
| "rewards/accuracies": 0.5249999761581421, |
| "rewards/chosen": -0.14577992260456085, |
| "rewards/margins": 0.009255652315914631, |
| "rewards/rejected": -0.15503555536270142, |
| "sft_loss": 1.4577990770339966, |
| "step": 500 |
| }, |
| { |
| "epoch": 0.8890864636585908, |
| "eval_logits/chosen": -21.487462997436523, |
| "eval_logits/rejected": -21.574512481689453, |
| "eval_logps/chosen": -1.3780959844589233, |
| "eval_logps/rejected": -1.5480619668960571, |
| "eval_loss": 1.4461547136306763, |
| "eval_odds_ratio_loss": 0.6805880665779114, |
| "eval_rewards/accuracies": 0.546999990940094, |
| "eval_rewards/chosen": -0.13780958950519562, |
| "eval_rewards/margins": 0.016996610909700394, |
| "eval_rewards/rejected": -0.1548061966896057, |
| "eval_runtime": 80.0397, |
| "eval_samples_per_second": 12.494, |
| "eval_sft_loss": 1.3780959844589233, |
| "eval_steps_per_second": 6.247, |
| "step": 500 |
| }, |
| { |
| "epoch": 0.9068681929317626, |
| "grad_norm": 2.9641082286834717, |
| "learning_rate": 3.953854170549114e-06, |
| "logits/chosen": -21.522262573242188, |
| "logits/rejected": -21.48137092590332, |
| "logps/chosen": -1.3978930711746216, |
| "logps/rejected": -1.4638049602508545, |
| "loss": 1.4702, |
| "odds_ratio_loss": 0.722897469997406, |
| "rewards/accuracies": 0.574999988079071, |
| "rewards/chosen": -0.13978929817676544, |
| "rewards/margins": 0.006591203156858683, |
| "rewards/rejected": -0.1463804990053177, |
| "sft_loss": 1.3978930711746216, |
| "step": 510 |
| }, |
| { |
| "epoch": 0.9246499222049345, |
| "grad_norm": 1.988604187965393, |
| "learning_rate": 3.91570523302051e-06, |
| "logits/chosen": -21.512929916381836, |
| "logits/rejected": -21.350711822509766, |
| "logps/chosen": -1.4139622449874878, |
| "logps/rejected": -1.502074122428894, |
| "loss": 1.4897, |
| "odds_ratio_loss": 0.7573299407958984, |
| "rewards/accuracies": 0.512499988079071, |
| "rewards/chosen": -0.14139625430107117, |
| "rewards/margins": 0.008811171166598797, |
| "rewards/rejected": -0.15020740032196045, |
| "sft_loss": 1.4139622449874878, |
| "step": 520 |
| }, |
| { |
| "epoch": 0.9424316514781063, |
| "grad_norm": 1.4567950963974, |
| "learning_rate": 3.8770647130141996e-06, |
| "logits/chosen": -21.612693786621094, |
| "logits/rejected": -21.457687377929688, |
| "logps/chosen": -1.3569138050079346, |
| "logps/rejected": -1.5465893745422363, |
| "loss": 1.4271, |
| "odds_ratio_loss": 0.7022345662117004, |
| "rewards/accuracies": 0.5249999761581421, |
| "rewards/chosen": -0.13569141924381256, |
| "rewards/margins": 0.01896754465997219, |
| "rewards/rejected": -0.1546589434146881, |
| "sft_loss": 1.3569138050079346, |
| "step": 530 |
| }, |
| { |
| "epoch": 0.960213380751278, |
| "grad_norm": 1.3677376508712769, |
| "learning_rate": 3.837946027873086e-06, |
| "logits/chosen": -21.51246452331543, |
| "logits/rejected": -21.367631912231445, |
| "logps/chosen": -1.4506080150604248, |
| "logps/rejected": -1.578880786895752, |
| "loss": 1.5258, |
| "odds_ratio_loss": 0.7515760660171509, |
| "rewards/accuracies": 0.5687500238418579, |
| "rewards/chosen": -0.14506080746650696, |
| "rewards/margins": 0.012827281840145588, |
| "rewards/rejected": -0.15788806974887848, |
| "sft_loss": 1.4506080150604248, |
| "step": 540 |
| }, |
| { |
| "epoch": 0.9779951100244498, |
| "grad_norm": 2.220174789428711, |
| "learning_rate": 3.7983627609757713e-06, |
| "logits/chosen": -21.598114013671875, |
| "logits/rejected": -21.58673095703125, |
| "logps/chosen": -1.4242851734161377, |
| "logps/rejected": -1.521756887435913, |
| "loss": 1.495, |
| "odds_ratio_loss": 0.7070504426956177, |
| "rewards/accuracies": 0.512499988079071, |
| "rewards/chosen": -0.14242851734161377, |
| "rewards/margins": 0.009747164323925972, |
| "rewards/rejected": -0.1521756947040558, |
| "sft_loss": 1.4242851734161377, |
| "step": 550 |
| }, |
| { |
| "epoch": 0.9957768392976217, |
| "grad_norm": 4.630890369415283, |
| "learning_rate": 3.758328657019924e-06, |
| "logits/chosen": -21.449283599853516, |
| "logits/rejected": -21.295719146728516, |
| "logps/chosen": -1.3235647678375244, |
| "logps/rejected": -1.4640743732452393, |
| "loss": 1.394, |
| "odds_ratio_loss": 0.7047213315963745, |
| "rewards/accuracies": 0.5375000238418579, |
| "rewards/chosen": -0.1323564499616623, |
| "rewards/margins": 0.014050972647964954, |
| "rewards/rejected": -0.14640744030475616, |
| "sft_loss": 1.3235647678375244, |
| "step": 560 |
| }, |
| { |
| "epoch": 1.0135585685707935, |
| "grad_norm": 4.50676155090332, |
| "learning_rate": 3.717857617249642e-06, |
| "logits/chosen": -21.255306243896484, |
| "logits/rejected": -21.378076553344727, |
| "logps/chosen": -1.4302732944488525, |
| "logps/rejected": -1.5925706624984741, |
| "loss": 1.5037, |
| "odds_ratio_loss": 0.7346171140670776, |
| "rewards/accuracies": 0.5062500238418579, |
| "rewards/chosen": -0.14302733540534973, |
| "rewards/margins": 0.016229737550020218, |
| "rewards/rejected": -0.15925706923007965, |
| "sft_loss": 1.4302732944488525, |
| "step": 570 |
| }, |
| { |
| "epoch": 1.0313402978439654, |
| "grad_norm": 1.0999839305877686, |
| "learning_rate": 3.6769636946284543e-06, |
| "logits/chosen": -21.335865020751953, |
| "logits/rejected": -21.204635620117188, |
| "logps/chosen": -1.2982518672943115, |
| "logps/rejected": -1.4139636754989624, |
| "loss": 1.3699, |
| "odds_ratio_loss": 0.7168216705322266, |
| "rewards/accuracies": 0.4625000059604645, |
| "rewards/chosen": -0.12982520461082458, |
| "rewards/margins": 0.011571166105568409, |
| "rewards/rejected": -0.14139637351036072, |
| "sft_loss": 1.2982518672943115, |
| "step": 580 |
| }, |
| { |
| "epoch": 1.049122027117137, |
| "grad_norm": 2.193345069885254, |
| "learning_rate": 3.6356610889596355e-06, |
| "logits/chosen": -21.441791534423828, |
| "logits/rejected": -21.434829711914062, |
| "logps/chosen": -1.371517539024353, |
| "logps/rejected": -1.4539680480957031, |
| "loss": 1.444, |
| "odds_ratio_loss": 0.724626362323761, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.13715174794197083, |
| "rewards/margins": 0.008245043456554413, |
| "rewards/rejected": -0.14539679884910583, |
| "sft_loss": 1.371517539024353, |
| "step": 590 |
| }, |
| { |
| "epoch": 1.066903756390309, |
| "grad_norm": 1.4549708366394043, |
| "learning_rate": 3.593964141955541e-06, |
| "logits/chosen": -21.462820053100586, |
| "logits/rejected": -21.234458923339844, |
| "logps/chosen": -1.355276107788086, |
| "logps/rejected": -1.4370988607406616, |
| "loss": 1.4288, |
| "odds_ratio_loss": 0.7349004149436951, |
| "rewards/accuracies": 0.512499988079071, |
| "rewards/chosen": -0.1355275958776474, |
| "rewards/margins": 0.008182285353541374, |
| "rewards/rejected": -0.14370988309383392, |
| "sft_loss": 1.355276107788086, |
| "step": 600 |
| }, |
| { |
| "epoch": 1.0846854856634809, |
| "grad_norm": 1.5671168565750122, |
| "learning_rate": 3.5518873322576573e-06, |
| "logits/chosen": -20.91110610961914, |
| "logits/rejected": -21.30324935913086, |
| "logps/chosen": -1.3413856029510498, |
| "logps/rejected": -1.4344289302825928, |
| "loss": 1.4148, |
| "odds_ratio_loss": 0.7339103817939758, |
| "rewards/accuracies": 0.48124998807907104, |
| "rewards/chosen": -0.1341385543346405, |
| "rewards/margins": 0.009304327890276909, |
| "rewards/rejected": -0.14344289898872375, |
| "sft_loss": 1.3413856029510498, |
| "step": 610 |
| }, |
| { |
| "epoch": 1.1024672149366526, |
| "grad_norm": 1.315266728401184, |
| "learning_rate": 3.5094452704091143e-06, |
| "logits/chosen": -21.287628173828125, |
| "logits/rejected": -21.211769104003906, |
| "logps/chosen": -1.3973969221115112, |
| "logps/rejected": -1.5142686367034912, |
| "loss": 1.4697, |
| "odds_ratio_loss": 0.7228156924247742, |
| "rewards/accuracies": 0.5062500238418579, |
| "rewards/chosen": -0.13973967730998993, |
| "rewards/margins": 0.01168716698884964, |
| "rewards/rejected": -0.15142686665058136, |
| "sft_loss": 1.3973969221115112, |
| "step": 620 |
| }, |
| { |
| "epoch": 1.1202489442098245, |
| "grad_norm": 3.814415693283081, |
| "learning_rate": 3.46665269378139e-06, |
| "logits/chosen": -21.241634368896484, |
| "logits/rejected": -21.107349395751953, |
| "logps/chosen": -1.4169210195541382, |
| "logps/rejected": -1.4841772317886353, |
| "loss": 1.4919, |
| "odds_ratio_loss": 0.7493141889572144, |
| "rewards/accuracies": 0.5062500238418579, |
| "rewards/chosen": -0.1416921317577362, |
| "rewards/margins": 0.006725601851940155, |
| "rewards/rejected": -0.14841774106025696, |
| "sft_loss": 1.4169210195541382, |
| "step": 630 |
| }, |
| { |
| "epoch": 1.1380306734829961, |
| "grad_norm": 5.05172872543335, |
| "learning_rate": 3.4235244614569794e-06, |
| "logits/chosen": -21.426654815673828, |
| "logits/rejected": -21.443878173828125, |
| "logps/chosen": -1.452530026435852, |
| "logps/rejected": -1.5365841388702393, |
| "loss": 1.528, |
| "odds_ratio_loss": 0.7546505928039551, |
| "rewards/accuracies": 0.512499988079071, |
| "rewards/chosen": -0.1452530175447464, |
| "rewards/margins": 0.00840541161596775, |
| "rewards/rejected": -0.1536584198474884, |
| "sft_loss": 1.452530026435852, |
| "step": 640 |
| }, |
| { |
| "epoch": 1.155812402756168, |
| "grad_norm": 1.0250731706619263, |
| "learning_rate": 3.3800755490698008e-06, |
| "logits/chosen": -21.588850021362305, |
| "logits/rejected": -21.425325393676758, |
| "logps/chosen": -1.321417212486267, |
| "logps/rejected": -1.539794921875, |
| "loss": 1.3862, |
| "odds_ratio_loss": 0.6476024985313416, |
| "rewards/accuracies": 0.550000011920929, |
| "rewards/chosen": -0.13214172422885895, |
| "rewards/margins": 0.021837763488292694, |
| "rewards/rejected": -0.15397948026657104, |
| "sft_loss": 1.321417212486267, |
| "step": 650 |
| }, |
| { |
| "epoch": 1.17359413202934, |
| "grad_norm": 1.5032236576080322, |
| "learning_rate": 3.3363210436051287e-06, |
| "logits/chosen": -21.39287567138672, |
| "logits/rejected": -21.30692481994629, |
| "logps/chosen": -1.409549593925476, |
| "logps/rejected": -1.5230066776275635, |
| "loss": 1.4823, |
| "odds_ratio_loss": 0.7271451950073242, |
| "rewards/accuracies": 0.5249999761581421, |
| "rewards/chosen": -0.14095497131347656, |
| "rewards/margins": 0.011345705017447472, |
| "rewards/rejected": -0.15230068564414978, |
| "sft_loss": 1.409549593925476, |
| "step": 660 |
| }, |
| { |
| "epoch": 1.1913758613025116, |
| "grad_norm": 1.4157321453094482, |
| "learning_rate": 3.292276138160867e-06, |
| "logits/chosen": -21.29572105407715, |
| "logits/rejected": -21.25027084350586, |
| "logps/chosen": -1.4160873889923096, |
| "logps/rejected": -1.496361255645752, |
| "loss": 1.491, |
| "odds_ratio_loss": 0.749149739742279, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.14160871505737305, |
| "rewards/margins": 0.00802740640938282, |
| "rewards/rejected": -0.14963611960411072, |
| "sft_loss": 1.4160873889923096, |
| "step": 670 |
| }, |
| { |
| "epoch": 1.2091575905756835, |
| "grad_norm": 1.573752522468567, |
| "learning_rate": 3.2479561266719694e-06, |
| "logits/chosen": -21.345748901367188, |
| "logits/rejected": -21.232250213623047, |
| "logps/chosen": -1.3891535997390747, |
| "logps/rejected": -1.5042526721954346, |
| "loss": 1.4603, |
| "odds_ratio_loss": 0.7116107940673828, |
| "rewards/accuracies": 0.5562499761581421, |
| "rewards/chosen": -0.13891534507274628, |
| "rewards/margins": 0.011509908363223076, |
| "rewards/rejected": -0.1504252701997757, |
| "sft_loss": 1.3891535997390747, |
| "step": 680 |
| }, |
| { |
| "epoch": 1.2269393198488552, |
| "grad_norm": 2.5362017154693604, |
| "learning_rate": 3.2033763985998533e-06, |
| "logits/chosen": -21.208703994750977, |
| "logits/rejected": -21.204181671142578, |
| "logps/chosen": -1.3326551914215088, |
| "logps/rejected": -1.5722543001174927, |
| "loss": 1.3975, |
| "odds_ratio_loss": 0.6482537984848022, |
| "rewards/accuracies": 0.5687500238418579, |
| "rewards/chosen": -0.13326552510261536, |
| "rewards/margins": 0.02395990863442421, |
| "rewards/rejected": -0.15722543001174927, |
| "sft_loss": 1.3326551914215088, |
| "step": 690 |
| }, |
| { |
| "epoch": 1.244721049122027, |
| "grad_norm": 1.910599946975708, |
| "learning_rate": 3.1585524335886335e-06, |
| "logits/chosen": -21.477584838867188, |
| "logits/rejected": -21.243457794189453, |
| "logps/chosen": -1.299839735031128, |
| "logps/rejected": -1.449894666671753, |
| "loss": 1.3692, |
| "odds_ratio_loss": 0.6940584182739258, |
| "rewards/accuracies": 0.512499988079071, |
| "rewards/chosen": -0.12998396158218384, |
| "rewards/margins": 0.015005489811301231, |
| "rewards/rejected": -0.14498946070671082, |
| "sft_loss": 1.299839735031128, |
| "step": 700 |
| }, |
| { |
| "epoch": 1.262502778395199, |
| "grad_norm": 2.3555686473846436, |
| "learning_rate": 3.1134997960900536e-06, |
| "logits/chosen": -20.757158279418945, |
| "logits/rejected": -20.784774780273438, |
| "logps/chosen": -1.2707315683364868, |
| "logps/rejected": -1.538629174232483, |
| "loss": 1.334, |
| "odds_ratio_loss": 0.6324664950370789, |
| "rewards/accuracies": 0.606249988079071, |
| "rewards/chosen": -0.12707316875457764, |
| "rewards/margins": 0.026789745315909386, |
| "rewards/rejected": -0.15386290848255157, |
| "sft_loss": 1.2707315683364868, |
| "step": 710 |
| }, |
| { |
| "epoch": 1.2802845076683709, |
| "grad_norm": 1.596251130104065, |
| "learning_rate": 3.0682341299589583e-06, |
| "logits/chosen": -20.7999324798584, |
| "logits/rejected": -20.802942276000977, |
| "logps/chosen": -1.3453733921051025, |
| "logps/rejected": -1.4210965633392334, |
| "loss": 1.4194, |
| "odds_ratio_loss": 0.7405422329902649, |
| "rewards/accuracies": 0.44999998807907104, |
| "rewards/chosen": -0.13453733921051025, |
| "rewards/margins": 0.0075723156332969666, |
| "rewards/rejected": -0.14210966229438782, |
| "sft_loss": 1.3453733921051025, |
| "step": 720 |
| }, |
| { |
| "epoch": 1.2980662369415426, |
| "grad_norm": 9.634610176086426, |
| "learning_rate": 3.022771153021201e-06, |
| "logits/chosen": -21.071128845214844, |
| "logits/rejected": -21.114444732666016, |
| "logps/chosen": -1.3551054000854492, |
| "logps/rejected": -1.5129811763763428, |
| "loss": 1.4248, |
| "odds_ratio_loss": 0.6970704197883606, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.13551053404808044, |
| "rewards/margins": 0.015787573531270027, |
| "rewards/rejected": -0.15129812061786652, |
| "sft_loss": 1.3551054000854492, |
| "step": 730 |
| }, |
| { |
| "epoch": 1.3158479662147144, |
| "grad_norm": 1.6211514472961426, |
| "learning_rate": 2.9771266516158625e-06, |
| "logits/chosen": -20.895437240600586, |
| "logits/rejected": -21.04778480529785, |
| "logps/chosen": -1.3368757963180542, |
| "logps/rejected": -1.4986876249313354, |
| "loss": 1.4062, |
| "odds_ratio_loss": 0.6928601264953613, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.1336875855922699, |
| "rewards/margins": 0.016181182116270065, |
| "rewards/rejected": -0.14986875653266907, |
| "sft_loss": 1.3368757963180542, |
| "step": 740 |
| }, |
| { |
| "epoch": 1.3336296954878861, |
| "grad_norm": 1.4428294897079468, |
| "learning_rate": 2.9313164751136802e-06, |
| "logits/chosen": -20.872339248657227, |
| "logits/rejected": -21.019441604614258, |
| "logps/chosen": -1.4122194051742554, |
| "logps/rejected": -1.487066626548767, |
| "loss": 1.4842, |
| "odds_ratio_loss": 0.7194846272468567, |
| "rewards/accuracies": 0.518750011920929, |
| "rewards/chosen": -0.14122194051742554, |
| "rewards/margins": 0.007484720554202795, |
| "rewards/rejected": -0.14870667457580566, |
| "sft_loss": 1.4122194051742554, |
| "step": 750 |
| }, |
| { |
| "epoch": 1.351411424761058, |
| "grad_norm": 2.2369892597198486, |
| "learning_rate": 2.8853565304135956e-06, |
| "logits/chosen": -21.462568283081055, |
| "logits/rejected": -21.10171127319336, |
| "logps/chosen": -1.352461576461792, |
| "logps/rejected": -1.3995507955551147, |
| "loss": 1.4271, |
| "odds_ratio_loss": 0.7464177012443542, |
| "rewards/accuracies": 0.48124998807907104, |
| "rewards/chosen": -0.1352461576461792, |
| "rewards/margins": 0.004708918742835522, |
| "rewards/rejected": -0.1399550884962082, |
| "sft_loss": 1.352461576461792, |
| "step": 760 |
| }, |
| { |
| "epoch": 1.36919315403423, |
| "grad_norm": 4.38085412979126, |
| "learning_rate": 2.839262776419313e-06, |
| "logits/chosen": -20.986604690551758, |
| "logits/rejected": -20.851150512695312, |
| "logps/chosen": -1.3386285305023193, |
| "logps/rejected": -1.5874344110488892, |
| "loss": 1.4054, |
| "odds_ratio_loss": 0.6678156852722168, |
| "rewards/accuracies": 0.5874999761581421, |
| "rewards/chosen": -0.13386288285255432, |
| "rewards/margins": 0.024880561977624893, |
| "rewards/rejected": -0.15874342620372772, |
| "sft_loss": 1.3386285305023193, |
| "step": 770 |
| }, |
| { |
| "epoch": 1.3869748833074016, |
| "grad_norm": 1.5637321472167969, |
| "learning_rate": 2.793051218497817e-06, |
| "logits/chosen": -21.250728607177734, |
| "logits/rejected": -21.10789680480957, |
| "logps/chosen": -1.3795894384384155, |
| "logps/rejected": -1.4174426794052124, |
| "loss": 1.4542, |
| "odds_ratio_loss": 0.7466022968292236, |
| "rewards/accuracies": 0.4749999940395355, |
| "rewards/chosen": -0.13795895874500275, |
| "rewards/margins": 0.003785322653129697, |
| "rewards/rejected": -0.14174428582191467, |
| "sft_loss": 1.3795894384384155, |
| "step": 780 |
| }, |
| { |
| "epoch": 1.4047566125805735, |
| "grad_norm": 1.276485800743103, |
| "learning_rate": 2.7467379029217437e-06, |
| "logits/chosen": -20.930208206176758, |
| "logits/rejected": -20.79922103881836, |
| "logps/chosen": -1.3249105215072632, |
| "logps/rejected": -1.5102876424789429, |
| "loss": 1.3942, |
| "odds_ratio_loss": 0.6923983693122864, |
| "rewards/accuracies": 0.550000011920929, |
| "rewards/chosen": -0.1324910670518875, |
| "rewards/margins": 0.018537694588303566, |
| "rewards/rejected": -0.15102875232696533, |
| "sft_loss": 1.3249105215072632, |
| "step": 790 |
| }, |
| { |
| "epoch": 1.4225383418537452, |
| "grad_norm": 1.1495212316513062, |
| "learning_rate": 2.7003389112975546e-06, |
| "logits/chosen": -21.19894027709961, |
| "logits/rejected": -21.32394790649414, |
| "logps/chosen": -1.3503518104553223, |
| "logps/rejected": -1.484006404876709, |
| "loss": 1.4195, |
| "odds_ratio_loss": 0.6918057203292847, |
| "rewards/accuracies": 0.4937500059604645, |
| "rewards/chosen": -0.1350351870059967, |
| "rewards/margins": 0.013365456834435463, |
| "rewards/rejected": -0.14840063452720642, |
| "sft_loss": 1.3503518104553223, |
| "step": 800 |
| }, |
| { |
| "epoch": 1.440320071126917, |
| "grad_norm": 4.020893573760986, |
| "learning_rate": 2.653870354981437e-06, |
| "logits/chosen": -21.07791519165039, |
| "logits/rejected": -20.885608673095703, |
| "logps/chosen": -1.2470946311950684, |
| "logps/rejected": -1.3942331075668335, |
| "loss": 1.3155, |
| "odds_ratio_loss": 0.6838669776916504, |
| "rewards/accuracies": 0.550000011920929, |
| "rewards/chosen": -0.12470944970846176, |
| "rewards/margins": 0.014713853597640991, |
| "rewards/rejected": -0.13942332565784454, |
| "sft_loss": 1.2470946311950684, |
| "step": 810 |
| }, |
| { |
| "epoch": 1.458101800400089, |
| "grad_norm": 3.656785011291504, |
| "learning_rate": 2.6073483694848777e-06, |
| "logits/chosen": -20.693532943725586, |
| "logits/rejected": -21.043460845947266, |
| "logps/chosen": -1.288588285446167, |
| "logps/rejected": -1.4409494400024414, |
| "loss": 1.3582, |
| "odds_ratio_loss": 0.6965524554252625, |
| "rewards/accuracies": 0.550000011920929, |
| "rewards/chosen": -0.12885884940624237, |
| "rewards/margins": 0.015236112289130688, |
| "rewards/rejected": -0.14409494400024414, |
| "sft_loss": 1.288588285446167, |
| "step": 820 |
| }, |
| { |
| "epoch": 1.4758835296732609, |
| "grad_norm": 1.309704303741455, |
| "learning_rate": 2.560789108871847e-06, |
| "logits/chosen": -20.856311798095703, |
| "logits/rejected": -20.888708114624023, |
| "logps/chosen": -1.3494679927825928, |
| "logps/rejected": -1.5807578563690186, |
| "loss": 1.4162, |
| "odds_ratio_loss": 0.6674301028251648, |
| "rewards/accuracies": 0.5062500238418579, |
| "rewards/chosen": -0.1349468231201172, |
| "rewards/margins": 0.02312898077070713, |
| "rewards/rejected": -0.15807577967643738, |
| "sft_loss": 1.3494679927825928, |
| "step": 830 |
| }, |
| { |
| "epoch": 1.4936652589464325, |
| "grad_norm": 5.437036037445068, |
| "learning_rate": 2.514208740149544e-06, |
| "logits/chosen": -21.045909881591797, |
| "logits/rejected": -21.26214599609375, |
| "logps/chosen": -1.4145755767822266, |
| "logps/rejected": -1.549298644065857, |
| "loss": 1.4864, |
| "odds_ratio_loss": 0.7186577320098877, |
| "rewards/accuracies": 0.5062500238418579, |
| "rewards/chosen": -0.14145755767822266, |
| "rewards/margins": 0.013472315855324268, |
| "rewards/rejected": -0.15492987632751465, |
| "sft_loss": 1.4145755767822266, |
| "step": 840 |
| }, |
| { |
| "epoch": 1.5114469882196042, |
| "grad_norm": 2.7086102962493896, |
| "learning_rate": 2.46762343765464e-06, |
| "logits/chosen": -21.045820236206055, |
| "logits/rejected": -21.116756439208984, |
| "logps/chosen": -1.4063694477081299, |
| "logps/rejected": -1.5858089923858643, |
| "loss": 1.4749, |
| "odds_ratio_loss": 0.6853106021881104, |
| "rewards/accuracies": 0.5249999761581421, |
| "rewards/chosen": -0.14063693583011627, |
| "rewards/margins": 0.01794394478201866, |
| "rewards/rejected": -0.15858088433742523, |
| "sft_loss": 1.4063694477081299, |
| "step": 850 |
| }, |
| { |
| "epoch": 1.5292287174927761, |
| "grad_norm": 3.8162646293640137, |
| "learning_rate": 2.4210493774369903e-06, |
| "logits/chosen": -20.788593292236328, |
| "logits/rejected": -20.716583251953125, |
| "logps/chosen": -1.406285285949707, |
| "logps/rejected": -1.5719993114471436, |
| "loss": 1.4773, |
| "odds_ratio_loss": 0.7099908590316772, |
| "rewards/accuracies": 0.518750011920929, |
| "rewards/chosen": -0.14062853157520294, |
| "rewards/margins": 0.016571396961808205, |
| "rewards/rejected": -0.1571999341249466, |
| "sft_loss": 1.406285285949707, |
| "step": 860 |
| }, |
| { |
| "epoch": 1.547010446765948, |
| "grad_norm": 1.2074657678604126, |
| "learning_rate": 2.374502731642732e-06, |
| "logits/chosen": -20.910995483398438, |
| "logits/rejected": -20.997020721435547, |
| "logps/chosen": -1.3468477725982666, |
| "logps/rejected": -1.490969181060791, |
| "loss": 1.4171, |
| "odds_ratio_loss": 0.7025480270385742, |
| "rewards/accuracies": 0.550000011920929, |
| "rewards/chosen": -0.13468477129936218, |
| "rewards/margins": 0.014412140473723412, |
| "rewards/rejected": -0.14909692108631134, |
| "sft_loss": 1.3468477725982666, |
| "step": 870 |
| }, |
| { |
| "epoch": 1.56479217603912, |
| "grad_norm": 1.6771084070205688, |
| "learning_rate": 2.3279996628987556e-06, |
| "logits/chosen": -21.090503692626953, |
| "logits/rejected": -21.15408706665039, |
| "logps/chosen": -1.3241709470748901, |
| "logps/rejected": -1.4298092126846313, |
| "loss": 1.397, |
| "odds_ratio_loss": 0.7280608415603638, |
| "rewards/accuracies": 0.5874999761581421, |
| "rewards/chosen": -0.13241711258888245, |
| "rewards/margins": 0.010563802905380726, |
| "rewards/rejected": -0.1429809182882309, |
| "sft_loss": 1.3241709470748901, |
| "step": 880 |
| }, |
| { |
| "epoch": 1.5825739053122916, |
| "grad_norm": 2.092092514038086, |
| "learning_rate": 2.281556318700474e-06, |
| "logits/chosen": -20.86192512512207, |
| "logits/rejected": -21.044658660888672, |
| "logps/chosen": -1.3072993755340576, |
| "logps/rejected": -1.3738138675689697, |
| "loss": 1.3822, |
| "odds_ratio_loss": 0.7485288381576538, |
| "rewards/accuracies": 0.4937500059604645, |
| "rewards/chosen": -0.13072994351387024, |
| "rewards/margins": 0.006651435978710651, |
| "rewards/rejected": -0.13738137483596802, |
| "sft_loss": 1.3072993755340576, |
| "step": 890 |
| }, |
| { |
| "epoch": 1.6003556345854635, |
| "grad_norm": 6.660823822021484, |
| "learning_rate": 2.2351888258048408e-06, |
| "logits/chosen": -20.55089569091797, |
| "logits/rejected": -20.74386978149414, |
| "logps/chosen": -1.3101674318313599, |
| "logps/rejected": -1.4409325122833252, |
| "loss": 1.3826, |
| "odds_ratio_loss": 0.7239800691604614, |
| "rewards/accuracies": 0.5562499761581421, |
| "rewards/chosen": -0.13101674616336823, |
| "rewards/margins": 0.013076506555080414, |
| "rewards/rejected": -0.14409324526786804, |
| "sft_loss": 1.3101674318313599, |
| "step": 900 |
| }, |
| { |
| "epoch": 1.6181373638586352, |
| "grad_norm": 1.3889552354812622, |
| "learning_rate": 2.188913284630584e-06, |
| "logits/chosen": -21.00895118713379, |
| "logits/rejected": -21.11439323425293, |
| "logps/chosen": -1.3723797798156738, |
| "logps/rejected": -1.4007512331008911, |
| "loss": 1.449, |
| "odds_ratio_loss": 0.7658642530441284, |
| "rewards/accuracies": 0.518750011920929, |
| "rewards/chosen": -0.13723799586296082, |
| "rewards/margins": 0.0028371470980346203, |
| "rewards/rejected": -0.14007511734962463, |
| "sft_loss": 1.3723797798156738, |
| "step": 910 |
| }, |
| { |
| "epoch": 1.635919093131807, |
| "grad_norm": 4.06219482421875, |
| "learning_rate": 2.1427457636675652e-06, |
| "logits/chosen": -21.082805633544922, |
| "logits/rejected": -21.207538604736328, |
| "logps/chosen": -1.336096167564392, |
| "logps/rejected": -1.4373667240142822, |
| "loss": 1.4096, |
| "odds_ratio_loss": 0.7348427176475525, |
| "rewards/accuracies": 0.48124998807907104, |
| "rewards/chosen": -0.13360963761806488, |
| "rewards/margins": 0.010127037763595581, |
| "rewards/rejected": -0.14373667538166046, |
| "sft_loss": 1.336096167564392, |
| "step": 920 |
| }, |
| { |
| "epoch": 1.653700822404979, |
| "grad_norm": 1.464863657951355, |
| "learning_rate": 2.096702293897247e-06, |
| "logits/chosen": -20.881576538085938, |
| "logits/rejected": -20.812564849853516, |
| "logps/chosen": -1.3259438276290894, |
| "logps/rejected": -1.5576345920562744, |
| "loss": 1.3925, |
| "odds_ratio_loss": 0.6658841371536255, |
| "rewards/accuracies": 0.543749988079071, |
| "rewards/chosen": -0.13259439170360565, |
| "rewards/margins": 0.02316909097135067, |
| "rewards/rejected": -0.15576346218585968, |
| "sft_loss": 1.3259438276290894, |
| "step": 930 |
| }, |
| { |
| "epoch": 1.6714825516781509, |
| "grad_norm": 2.923940420150757, |
| "learning_rate": 2.0507988632261672e-06, |
| "logits/chosen": -20.792316436767578, |
| "logits/rejected": -20.86935043334961, |
| "logps/chosen": -1.3512170314788818, |
| "logps/rejected": -1.4976381063461304, |
| "loss": 1.4213, |
| "odds_ratio_loss": 0.7008249163627625, |
| "rewards/accuracies": 0.543749988079071, |
| "rewards/chosen": -0.1351216733455658, |
| "rewards/margins": 0.014642128720879555, |
| "rewards/rejected": -0.14976383745670319, |
| "sft_loss": 1.3512170314788818, |
| "step": 940 |
| }, |
| { |
| "epoch": 1.6892642809513225, |
| "grad_norm": 3.5238471031188965, |
| "learning_rate": 2.005051410934382e-06, |
| "logits/chosen": -20.95963478088379, |
| "logits/rejected": -20.97479248046875, |
| "logps/chosen": -1.4458208084106445, |
| "logps/rejected": -1.5041887760162354, |
| "loss": 1.5203, |
| "odds_ratio_loss": 0.7446683645248413, |
| "rewards/accuracies": 0.48750001192092896, |
| "rewards/chosen": -0.14458207786083221, |
| "rewards/margins": 0.005836788564920425, |
| "rewards/rejected": -0.15041887760162354, |
| "sft_loss": 1.4458208084106445, |
| "step": 950 |
| }, |
| { |
| "epoch": 1.7070460102244942, |
| "grad_norm": 2.6721088886260986, |
| "learning_rate": 1.9594758221407843e-06, |
| "logits/chosen": -20.884212493896484, |
| "logits/rejected": -20.890071868896484, |
| "logps/chosen": -1.3164139986038208, |
| "logps/rejected": -1.506830096244812, |
| "loss": 1.3826, |
| "odds_ratio_loss": 0.6615261435508728, |
| "rewards/accuracies": 0.574999988079071, |
| "rewards/chosen": -0.13164140284061432, |
| "rewards/margins": 0.019041623920202255, |
| "rewards/rejected": -0.15068301558494568, |
| "sft_loss": 1.3164139986038208, |
| "step": 960 |
| }, |
| { |
| "epoch": 1.724827739497666, |
| "grad_norm": 2.1806442737579346, |
| "learning_rate": 1.9140879222872408e-06, |
| "logits/chosen": -20.64748191833496, |
| "logits/rejected": -20.80613136291504, |
| "logps/chosen": -1.3217017650604248, |
| "logps/rejected": -1.4095408916473389, |
| "loss": 1.3952, |
| "odds_ratio_loss": 0.7351614236831665, |
| "rewards/accuracies": 0.5562499761581421, |
| "rewards/chosen": -0.1321701854467392, |
| "rewards/margins": 0.008783898316323757, |
| "rewards/rejected": -0.14095407724380493, |
| "sft_loss": 1.3217017650604248, |
| "step": 970 |
| }, |
| { |
| "epoch": 1.742609468770838, |
| "grad_norm": 1.7276735305786133, |
| "learning_rate": 1.8689034716434346e-06, |
| "logits/chosen": -21.096982955932617, |
| "logits/rejected": -21.016551971435547, |
| "logps/chosen": -1.3933743238449097, |
| "logps/rejected": -1.4783251285552979, |
| "loss": 1.4665, |
| "odds_ratio_loss": 0.7310749292373657, |
| "rewards/accuracies": 0.5062500238418579, |
| "rewards/chosen": -0.1393374353647232, |
| "rewards/margins": 0.008495080284774303, |
| "rewards/rejected": -0.14783251285552979, |
| "sft_loss": 1.3933743238449097, |
| "step": 980 |
| }, |
| { |
| "epoch": 1.76039119804401, |
| "grad_norm": 1.8123304843902588, |
| "learning_rate": 1.8239381598343576e-06, |
| "logits/chosen": -20.750640869140625, |
| "logits/rejected": -20.75037956237793, |
| "logps/chosen": -1.3481253385543823, |
| "logps/rejected": -1.4228088855743408, |
| "loss": 1.4223, |
| "odds_ratio_loss": 0.7416225671768188, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.13481254875659943, |
| "rewards/margins": 0.0074683367274701595, |
| "rewards/rejected": -0.14228087663650513, |
| "sft_loss": 1.3481253385543823, |
| "step": 990 |
| }, |
| { |
| "epoch": 1.7781729273171816, |
| "grad_norm": 3.5161044597625732, |
| "learning_rate": 1.779207600392312e-06, |
| "logits/chosen": -21.101184844970703, |
| "logits/rejected": -21.04250717163086, |
| "logps/chosen": -1.4057555198669434, |
| "logps/rejected": -1.5241641998291016, |
| "loss": 1.4761, |
| "odds_ratio_loss": 0.7033491134643555, |
| "rewards/accuracies": 0.5249999761581421, |
| "rewards/chosen": -0.1405755579471588, |
| "rewards/margins": 0.011840855702757835, |
| "rewards/rejected": -0.1524164229631424, |
| "sft_loss": 1.4057555198669434, |
| "step": 1000 |
| }, |
| { |
| "epoch": 1.7781729273171816, |
| "eval_logits/chosen": -20.803815841674805, |
| "eval_logits/rejected": -20.891420364379883, |
| "eval_logps/chosen": -1.3395137786865234, |
| "eval_logps/rejected": -1.5095441341400146, |
| "eval_loss": 1.407221794128418, |
| "eval_odds_ratio_loss": 0.6770801544189453, |
| "eval_rewards/accuracies": 0.5350000262260437, |
| "eval_rewards/chosen": -0.1339513659477234, |
| "eval_rewards/margins": 0.017003033310174942, |
| "eval_rewards/rejected": -0.15095441043376923, |
| "eval_runtime": 80.0538, |
| "eval_samples_per_second": 12.492, |
| "eval_sft_loss": 1.3395137786865234, |
| "eval_steps_per_second": 6.246, |
| "step": 1000 |
| }, |
| { |
| "epoch": 1.7959546565903532, |
| "grad_norm": 3.0343945026397705, |
| "learning_rate": 1.7347273253353552e-06, |
| "logits/chosen": -20.704559326171875, |
| "logits/rejected": -20.68727684020996, |
| "logps/chosen": -1.314007043838501, |
| "logps/rejected": -1.4146376848220825, |
| "loss": 1.389, |
| "odds_ratio_loss": 0.7499723434448242, |
| "rewards/accuracies": 0.4937500059604645, |
| "rewards/chosen": -0.1314007043838501, |
| "rewards/margins": 0.01006306428462267, |
| "rewards/rejected": -0.14146378636360168, |
| "sft_loss": 1.314007043838501, |
| "step": 1010 |
| }, |
| { |
| "epoch": 1.8137363858635251, |
| "grad_norm": 6.905886650085449, |
| "learning_rate": 1.690512779774029e-06, |
| "logits/chosen": -20.81467056274414, |
| "logits/rejected": -20.834705352783203, |
| "logps/chosen": -1.4023054838180542, |
| "logps/rejected": -1.652772307395935, |
| "loss": 1.4691, |
| "odds_ratio_loss": 0.6682060956954956, |
| "rewards/accuracies": 0.5874999761581421, |
| "rewards/chosen": -0.14023055136203766, |
| "rewards/margins": 0.025046680122613907, |
| "rewards/rejected": -0.16527722775936127, |
| "sft_loss": 1.4023054838180542, |
| "step": 1020 |
| }, |
| { |
| "epoch": 1.831518115136697, |
| "grad_norm": 1.7513582706451416, |
| "learning_rate": 1.6465793165482838e-06, |
| "logits/chosen": -20.86380386352539, |
| "logits/rejected": -20.82488441467285, |
| "logps/chosen": -1.2588412761688232, |
| "logps/rejected": -1.4600279331207275, |
| "loss": 1.3236, |
| "odds_ratio_loss": 0.6471126079559326, |
| "rewards/accuracies": 0.5874999761581421, |
| "rewards/chosen": -0.12588414549827576, |
| "rewards/margins": 0.02011866495013237, |
| "rewards/rejected": -0.14600279927253723, |
| "sft_loss": 1.2588412761688232, |
| "step": 1030 |
| }, |
| { |
| "epoch": 1.849299844409869, |
| "grad_norm": 2.2100415229797363, |
| "learning_rate": 1.6029421908964305e-06, |
| "logits/chosen": -20.778493881225586, |
| "logits/rejected": -20.64494514465332, |
| "logps/chosen": -1.3084795475006104, |
| "logps/rejected": -1.622815728187561, |
| "loss": 1.375, |
| "odds_ratio_loss": 0.664787232875824, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.1308479607105255, |
| "rewards/margins": 0.03143361583352089, |
| "rewards/rejected": -0.1622815579175949, |
| "sft_loss": 1.3084795475006104, |
| "step": 1040 |
| }, |
| { |
| "epoch": 1.8670815736830408, |
| "grad_norm": 4.5934367179870605, |
| "learning_rate": 1.559616555157985e-06, |
| "logits/chosen": -21.22179412841797, |
| "logits/rejected": -20.979957580566406, |
| "logps/chosen": -1.3719347715377808, |
| "logps/rejected": -1.5128790140151978, |
| "loss": 1.4452, |
| "odds_ratio_loss": 0.7331027388572693, |
| "rewards/accuracies": 0.543749988079071, |
| "rewards/chosen": -0.1371934711933136, |
| "rewards/margins": 0.014094437472522259, |
| "rewards/rejected": -0.15128789842128754, |
| "sft_loss": 1.3719347715377808, |
| "step": 1050 |
| }, |
| { |
| "epoch": 1.8848633029562125, |
| "grad_norm": 1.7163333892822266, |
| "learning_rate": 1.516617453512252e-06, |
| "logits/chosen": -21.102188110351562, |
| "logits/rejected": -21.131221771240234, |
| "logps/chosen": -1.4223716259002686, |
| "logps/rejected": -1.4797694683074951, |
| "loss": 1.498, |
| "odds_ratio_loss": 0.7560666799545288, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.14223715662956238, |
| "rewards/margins": 0.005739795975387096, |
| "rewards/rejected": -0.14797696471214294, |
| "sft_loss": 1.4223716259002686, |
| "step": 1060 |
| }, |
| { |
| "epoch": 1.9026450322293842, |
| "grad_norm": 9.351452827453613, |
| "learning_rate": 1.473959816754449e-06, |
| "logits/chosen": -20.615371704101562, |
| "logits/rejected": -20.649810791015625, |
| "logps/chosen": -1.3047014474868774, |
| "logps/rejected": -1.3762633800506592, |
| "loss": 1.3786, |
| "odds_ratio_loss": 0.7393638491630554, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.1304701417684555, |
| "rewards/margins": 0.00715619046241045, |
| "rewards/rejected": -0.13762633502483368, |
| "sft_loss": 1.3047014474868774, |
| "step": 1070 |
| }, |
| { |
| "epoch": 1.920426761502556, |
| "grad_norm": 2.190560817718506, |
| "learning_rate": 1.4316584571112213e-06, |
| "logits/chosen": -21.255840301513672, |
| "logits/rejected": -21.131498336791992, |
| "logps/chosen": -1.3643953800201416, |
| "logps/rejected": -1.461114525794983, |
| "loss": 1.4364, |
| "odds_ratio_loss": 0.7197447419166565, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.13643954694271088, |
| "rewards/margins": 0.009671924635767937, |
| "rewards/rejected": -0.14611145853996277, |
| "sft_loss": 1.3643953800201416, |
| "step": 1080 |
| }, |
| { |
| "epoch": 1.938208490775728, |
| "grad_norm": 2.542182683944702, |
| "learning_rate": 1.389728063097306e-06, |
| "logits/chosen": -20.93314552307129, |
| "logits/rejected": -20.863218307495117, |
| "logps/chosen": -1.389034390449524, |
| "logps/rejected": -1.5989328622817993, |
| "loss": 1.4573, |
| "odds_ratio_loss": 0.6827085614204407, |
| "rewards/accuracies": 0.543749988079071, |
| "rewards/chosen": -0.1389034539461136, |
| "rewards/margins": 0.020989837124943733, |
| "rewards/rejected": -0.15989328920841217, |
| "sft_loss": 1.389034390449524, |
| "step": 1090 |
| }, |
| { |
| "epoch": 1.9559902200488999, |
| "grad_norm": 1.5650415420532227, |
| "learning_rate": 1.348183194415179e-06, |
| "logits/chosen": -20.95106315612793, |
| "logits/rejected": -20.61818504333496, |
| "logps/chosen": -1.323676347732544, |
| "logps/rejected": -1.5667550563812256, |
| "loss": 1.3885, |
| "odds_ratio_loss": 0.6487289071083069, |
| "rewards/accuracies": 0.574999988079071, |
| "rewards/chosen": -0.13236764073371887, |
| "rewards/margins": 0.02430787682533264, |
| "rewards/rejected": -0.1566755324602127, |
| "sft_loss": 1.323676347732544, |
| "step": 1100 |
| }, |
| { |
| "epoch": 1.9737719493220716, |
| "grad_norm": 1.7203210592269897, |
| "learning_rate": 1.3070382768994015e-06, |
| "logits/chosen": -20.69628143310547, |
| "logits/rejected": -20.650815963745117, |
| "logps/chosen": -1.3079763650894165, |
| "logps/rejected": -1.449339747428894, |
| "loss": 1.3762, |
| "odds_ratio_loss": 0.6826270818710327, |
| "rewards/accuracies": 0.5062500238418579, |
| "rewards/chosen": -0.13079765439033508, |
| "rewards/margins": 0.014136332087218761, |
| "rewards/rejected": -0.14493396878242493, |
| "sft_loss": 1.3079763650894165, |
| "step": 1110 |
| }, |
| { |
| "epoch": 1.9915536785952432, |
| "grad_norm": 1.5598257780075073, |
| "learning_rate": 1.2663075975074746e-06, |
| "logits/chosen": -20.689380645751953, |
| "logits/rejected": -20.69732666015625, |
| "logps/chosen": -1.3402197360992432, |
| "logps/rejected": -1.503177285194397, |
| "loss": 1.4129, |
| "odds_ratio_loss": 0.7268449664115906, |
| "rewards/accuracies": 0.518750011920929, |
| "rewards/chosen": -0.13402198255062103, |
| "rewards/margins": 0.016295749694108963, |
| "rewards/rejected": -0.1503177136182785, |
| "sft_loss": 1.3402197360992432, |
| "step": 1120 |
| }, |
| { |
| "epoch": 2.009335407868415, |
| "grad_norm": 5.007309436798096, |
| "learning_rate": 1.2260052993589034e-06, |
| "logits/chosen": -20.855276107788086, |
| "logits/rejected": -20.814468383789062, |
| "logps/chosen": -1.4246357679367065, |
| "logps/rejected": -1.4585391283035278, |
| "loss": 1.5014, |
| "odds_ratio_loss": 0.7673634886741638, |
| "rewards/accuracies": 0.4749999940395355, |
| "rewards/chosen": -0.1424635797739029, |
| "rewards/margins": 0.003390337573364377, |
| "rewards/rejected": -0.1458539217710495, |
| "sft_loss": 1.4246357679367065, |
| "step": 1130 |
| }, |
| { |
| "epoch": 2.027117137141587, |
| "grad_norm": 1.2563971281051636, |
| "learning_rate": 1.1861453768242099e-06, |
| "logits/chosen": -20.794506072998047, |
| "logits/rejected": -20.795894622802734, |
| "logps/chosen": -1.2917953729629517, |
| "logps/rejected": -1.483782172203064, |
| "loss": 1.3577, |
| "odds_ratio_loss": 0.6590424180030823, |
| "rewards/accuracies": 0.574999988079071, |
| "rewards/chosen": -0.12917952239513397, |
| "rewards/margins": 0.019198691472411156, |
| "rewards/rejected": -0.14837822318077087, |
| "sft_loss": 1.2917953729629517, |
| "step": 1140 |
| }, |
| { |
| "epoch": 2.044898866414759, |
| "grad_norm": 8.363728523254395, |
| "learning_rate": 1.1467416706655982e-06, |
| "logits/chosen": -20.971622467041016, |
| "logits/rejected": -21.218524932861328, |
| "logps/chosen": -1.418050765991211, |
| "logps/rejected": -1.580128788948059, |
| "loss": 1.4924, |
| "odds_ratio_loss": 0.7438761591911316, |
| "rewards/accuracies": 0.5062500238418579, |
| "rewards/chosen": -0.14180508255958557, |
| "rewards/margins": 0.016207797452807426, |
| "rewards/rejected": -0.15801288187503815, |
| "sft_loss": 1.418050765991211, |
| "step": 1150 |
| }, |
| { |
| "epoch": 2.062680595687931, |
| "grad_norm": 1.5622318983078003, |
| "learning_rate": 1.1078078632309559e-06, |
| "logits/chosen": -20.65304946899414, |
| "logits/rejected": -20.797122955322266, |
| "logps/chosen": -1.3177438974380493, |
| "logps/rejected": -1.4770663976669312, |
| "loss": 1.3849, |
| "odds_ratio_loss": 0.6711241006851196, |
| "rewards/accuracies": 0.5687500238418579, |
| "rewards/chosen": -0.1317743957042694, |
| "rewards/margins": 0.015932243317365646, |
| "rewards/rejected": -0.14770662784576416, |
| "sft_loss": 1.3177438974380493, |
| "step": 1160 |
| }, |
| { |
| "epoch": 2.0804623249611023, |
| "grad_norm": 1.6626743078231812, |
| "learning_rate": 1.0693574737028627e-06, |
| "logits/chosen": -20.749677658081055, |
| "logits/rejected": -20.718555450439453, |
| "logps/chosen": -1.354952096939087, |
| "logps/rejected": -1.4761542081832886, |
| "loss": 1.428, |
| "odds_ratio_loss": 0.73005211353302, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.13549521565437317, |
| "rewards/margins": 0.012120204977691174, |
| "rewards/rejected": -0.14761541783809662, |
| "sft_loss": 1.354952096939087, |
| "step": 1170 |
| }, |
| { |
| "epoch": 2.098244054234274, |
| "grad_norm": 3.4684457778930664, |
| "learning_rate": 1.0314038534042586e-06, |
| "logits/chosen": -20.948108673095703, |
| "logits/rejected": -20.716609954833984, |
| "logps/chosen": -1.2302569150924683, |
| "logps/rejected": -1.4216673374176025, |
| "loss": 1.2983, |
| "odds_ratio_loss": 0.6800249814987183, |
| "rewards/accuracies": 0.550000011920929, |
| "rewards/chosen": -0.12302567809820175, |
| "rewards/margins": 0.019141051918268204, |
| "rewards/rejected": -0.14216673374176025, |
| "sft_loss": 1.2302569150924683, |
| "step": 1180 |
| }, |
| { |
| "epoch": 2.116025783507446, |
| "grad_norm": 1.7580640316009521, |
| "learning_rate": 9.939601811623946e-07, |
| "logits/chosen": -20.846065521240234, |
| "logits/rejected": -20.80862045288086, |
| "logps/chosen": -1.3318583965301514, |
| "logps/rejected": -1.4921131134033203, |
| "loss": 1.4031, |
| "odds_ratio_loss": 0.7127273678779602, |
| "rewards/accuracies": 0.5375000238418579, |
| "rewards/chosen": -0.13318583369255066, |
| "rewards/margins": 0.016025487333536148, |
| "rewards/rejected": -0.1492113173007965, |
| "sft_loss": 1.3318583965301514, |
| "step": 1190 |
| }, |
| { |
| "epoch": 2.133807512780618, |
| "grad_norm": 2.0461864471435547, |
| "learning_rate": 9.570394587326825e-07, |
| "logits/chosen": -21.051130294799805, |
| "logits/rejected": -20.864850997924805, |
| "logps/chosen": -1.322939395904541, |
| "logps/rejected": -1.5531421899795532, |
| "loss": 1.3893, |
| "odds_ratio_loss": 0.6637840867042542, |
| "rewards/accuracies": 0.5687500238418579, |
| "rewards/chosen": -0.1322939246892929, |
| "rewards/margins": 0.023020274937152863, |
| "rewards/rejected": -0.15531422197818756, |
| "sft_loss": 1.322939395904541, |
| "step": 1200 |
| }, |
| { |
| "epoch": 2.15158924205379, |
| "grad_norm": 1.2578119039535522, |
| "learning_rate": 9.206545062840302e-07, |
| "logits/chosen": -21.234752655029297, |
| "logits/rejected": -20.857492446899414, |
| "logps/chosen": -1.2849655151367188, |
| "logps/rejected": -1.4767402410507202, |
| "loss": 1.3511, |
| "odds_ratio_loss": 0.6610640287399292, |
| "rewards/accuracies": 0.581250011920929, |
| "rewards/chosen": -0.12849654257297516, |
| "rewards/margins": 0.019177492707967758, |
| "rewards/rejected": -0.14767403900623322, |
| "sft_loss": 1.2849655151367188, |
| "step": 1210 |
| }, |
| { |
| "epoch": 2.1693709713269618, |
| "grad_norm": 1.5944854021072388, |
| "learning_rate": 8.848179579472285e-07, |
| "logits/chosen": -20.92203140258789, |
| "logits/rejected": -20.818485260009766, |
| "logps/chosen": -1.2799731492996216, |
| "logps/rejected": -1.326030969619751, |
| "loss": 1.3534, |
| "odds_ratio_loss": 0.7338781952857971, |
| "rewards/accuracies": 0.4937500059604645, |
| "rewards/chosen": -0.12799732387065887, |
| "rewards/margins": 0.0046057915315032005, |
| "rewards/rejected": -0.13260310888290405, |
| "sft_loss": 1.2799731492996216, |
| "step": 1220 |
| }, |
| { |
| "epoch": 2.1871527006001332, |
| "grad_norm": 3.263883352279663, |
| "learning_rate": 8.495422574279403e-07, |
| "logits/chosen": -20.327661514282227, |
| "logits/rejected": -20.28653907775879, |
| "logps/chosen": -1.262486219406128, |
| "logps/rejected": -1.4986459016799927, |
| "loss": 1.3277, |
| "odds_ratio_loss": 0.6521891951560974, |
| "rewards/accuracies": 0.643750011920929, |
| "rewards/chosen": -0.12624862790107727, |
| "rewards/margins": 0.02361595258116722, |
| "rewards/rejected": -0.1498645842075348, |
| "sft_loss": 1.262486219406128, |
| "step": 1230 |
| }, |
| { |
| "epoch": 2.204934429873305, |
| "grad_norm": 2.0986313819885254, |
| "learning_rate": 8.148396536858063e-07, |
| "logits/chosen": -21.014957427978516, |
| "logits/rejected": -21.147602081298828, |
| "logps/chosen": -1.3925727605819702, |
| "logps/rejected": -1.589383840560913, |
| "loss": 1.463, |
| "odds_ratio_loss": 0.7045022249221802, |
| "rewards/accuracies": 0.5562499761581421, |
| "rewards/chosen": -0.1392572671175003, |
| "rewards/margins": 0.019681129604578018, |
| "rewards/rejected": -0.15893837809562683, |
| "sft_loss": 1.3925727605819702, |
| "step": 1240 |
| }, |
| { |
| "epoch": 2.222716159146477, |
| "grad_norm": 1.3406250476837158, |
| "learning_rate": 7.807221966811815e-07, |
| "logits/chosen": -20.607036590576172, |
| "logits/rejected": -20.66307830810547, |
| "logps/chosen": -1.2920827865600586, |
| "logps/rejected": -1.41164231300354, |
| "loss": 1.3638, |
| "odds_ratio_loss": 0.7169677019119263, |
| "rewards/accuracies": 0.4937500059604645, |
| "rewards/chosen": -0.1292082816362381, |
| "rewards/margins": 0.01195596344769001, |
| "rewards/rejected": -0.14116425812244415, |
| "sft_loss": 1.2920827865600586, |
| "step": 1250 |
| }, |
| { |
| "epoch": 2.240497888419649, |
| "grad_norm": 2.084696054458618, |
| "learning_rate": 7.47201733190962e-07, |
| "logits/chosen": -20.630435943603516, |
| "logits/rejected": -20.60986328125, |
| "logps/chosen": -1.2978394031524658, |
| "logps/rejected": -1.403597116470337, |
| "loss": 1.3682, |
| "odds_ratio_loss": 0.7040928602218628, |
| "rewards/accuracies": 0.48750001192092896, |
| "rewards/chosen": -0.12978394329547882, |
| "rewards/margins": 0.010575750842690468, |
| "rewards/rejected": -0.14035969972610474, |
| "sft_loss": 1.2978394031524658, |
| "step": 1260 |
| }, |
| { |
| "epoch": 2.258279617692821, |
| "grad_norm": 1.7839128971099854, |
| "learning_rate": 7.142899026949721e-07, |
| "logits/chosen": -20.951190948486328, |
| "logits/rejected": -20.935705184936523, |
| "logps/chosen": -1.320299744606018, |
| "logps/rejected": -1.4439074993133545, |
| "loss": 1.3892, |
| "odds_ratio_loss": 0.6885126233100891, |
| "rewards/accuracies": 0.574999988079071, |
| "rewards/chosen": -0.13202998042106628, |
| "rewards/margins": 0.012360776774585247, |
| "rewards/rejected": -0.1443907469511032, |
| "sft_loss": 1.320299744606018, |
| "step": 1270 |
| }, |
| { |
| "epoch": 2.2760613469659923, |
| "grad_norm": 7.137161731719971, |
| "learning_rate": 6.819981333343273e-07, |
| "logits/chosen": -20.221033096313477, |
| "logits/rejected": -20.27283477783203, |
| "logps/chosen": -1.2987910509109497, |
| "logps/rejected": -1.4801701307296753, |
| "loss": 1.3676, |
| "odds_ratio_loss": 0.6884258985519409, |
| "rewards/accuracies": 0.543749988079071, |
| "rewards/chosen": -0.12987910211086273, |
| "rewards/margins": 0.01813790202140808, |
| "rewards/rejected": -0.148017019033432, |
| "sft_loss": 1.2987910509109497, |
| "step": 1280 |
| }, |
| { |
| "epoch": 2.293843076239164, |
| "grad_norm": 2.513110876083374, |
| "learning_rate": 6.503376379431839e-07, |
| "logits/chosen": -20.69548797607422, |
| "logits/rejected": -20.705198287963867, |
| "logps/chosen": -1.4108153581619263, |
| "logps/rejected": -1.396875262260437, |
| "loss": 1.4867, |
| "odds_ratio_loss": 0.7589144110679626, |
| "rewards/accuracies": 0.48124998807907104, |
| "rewards/chosen": -0.1410815417766571, |
| "rewards/margins": -0.0013940061908215284, |
| "rewards/rejected": -0.13968753814697266, |
| "sft_loss": 1.4108153581619263, |
| "step": 1290 |
| }, |
| { |
| "epoch": 2.311624805512336, |
| "grad_norm": 6.825961112976074, |
| "learning_rate": 6.193194101552502e-07, |
| "logits/chosen": -20.706968307495117, |
| "logits/rejected": -20.34494400024414, |
| "logps/chosen": -1.327618956565857, |
| "logps/rejected": -1.4798409938812256, |
| "loss": 1.3947, |
| "odds_ratio_loss": 0.6709089279174805, |
| "rewards/accuracies": 0.550000011920929, |
| "rewards/chosen": -0.1327619105577469, |
| "rewards/margins": 0.01522219367325306, |
| "rewards/rejected": -0.147984117269516, |
| "sft_loss": 1.327618956565857, |
| "step": 1300 |
| }, |
| { |
| "epoch": 2.329406534785508, |
| "grad_norm": 2.9888756275177, |
| "learning_rate": 5.889542205864083e-07, |
| "logits/chosen": -20.558048248291016, |
| "logits/rejected": -20.51228904724121, |
| "logps/chosen": -1.3237196207046509, |
| "logps/rejected": -1.4659796953201294, |
| "loss": 1.3931, |
| "odds_ratio_loss": 0.6935244798660278, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.13237197697162628, |
| "rewards/margins": 0.014225991442799568, |
| "rewards/rejected": -0.1465979516506195, |
| "sft_loss": 1.3237196207046509, |
| "step": 1310 |
| }, |
| { |
| "epoch": 2.34718826405868, |
| "grad_norm": 1.8925628662109375, |
| "learning_rate": 5.592526130947862e-07, |
| "logits/chosen": -20.927398681640625, |
| "logits/rejected": -20.855573654174805, |
| "logps/chosen": -1.3616701364517212, |
| "logps/rejected": -1.4716918468475342, |
| "loss": 1.4372, |
| "odds_ratio_loss": 0.7552787065505981, |
| "rewards/accuracies": 0.4437499940395355, |
| "rewards/chosen": -0.1361670196056366, |
| "rewards/margins": 0.011002160608768463, |
| "rewards/rejected": -0.14716917276382446, |
| "sft_loss": 1.3616701364517212, |
| "step": 1320 |
| }, |
| { |
| "epoch": 2.3649699933318518, |
| "grad_norm": 2.2592906951904297, |
| "learning_rate": 5.302249011195507e-07, |
| "logits/chosen": -20.643238067626953, |
| "logits/rejected": -20.706254959106445, |
| "logps/chosen": -1.286387324333191, |
| "logps/rejected": -1.3748500347137451, |
| "loss": 1.3577, |
| "odds_ratio_loss": 0.712990939617157, |
| "rewards/accuracies": 0.518750011920929, |
| "rewards/chosen": -0.12863874435424805, |
| "rewards/margins": 0.008846262469887733, |
| "rewards/rejected": -0.13748499751091003, |
| "sft_loss": 1.286387324333191, |
| "step": 1330 |
| }, |
| { |
| "epoch": 2.382751722605023, |
| "grad_norm": 3.97871470451355, |
| "learning_rate": 5.018811640997307e-07, |
| "logits/chosen": -20.570959091186523, |
| "logits/rejected": -20.81188201904297, |
| "logps/chosen": -1.383998155593872, |
| "logps/rejected": -1.646945595741272, |
| "loss": 1.4509, |
| "odds_ratio_loss": 0.6689848899841309, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.13839980959892273, |
| "rewards/margins": 0.026294732466340065, |
| "rewards/rejected": -0.16469456255435944, |
| "sft_loss": 1.383998155593872, |
| "step": 1340 |
| }, |
| { |
| "epoch": 2.400533451878195, |
| "grad_norm": 1.2727420330047607, |
| "learning_rate": 4.7423124397427105e-07, |
| "logits/chosen": -20.430959701538086, |
| "logits/rejected": -20.650379180908203, |
| "logps/chosen": -1.35360848903656, |
| "logps/rejected": -1.4245867729187012, |
| "loss": 1.4266, |
| "odds_ratio_loss": 0.7302565574645996, |
| "rewards/accuracies": 0.4937500059604645, |
| "rewards/chosen": -0.13536083698272705, |
| "rewards/margins": 0.007097836583852768, |
| "rewards/rejected": -0.1424586921930313, |
| "sft_loss": 1.35360848903656, |
| "step": 1350 |
| }, |
| { |
| "epoch": 2.418315181151367, |
| "grad_norm": 2.574122428894043, |
| "learning_rate": 4.472847417645787e-07, |
| "logits/chosen": -20.755605697631836, |
| "logits/rejected": -20.458105087280273, |
| "logps/chosen": -1.3647658824920654, |
| "logps/rejected": -1.634526252746582, |
| "loss": 1.4294, |
| "odds_ratio_loss": 0.6465052366256714, |
| "rewards/accuracies": 0.5874999761581421, |
| "rewards/chosen": -0.13647659122943878, |
| "rewards/margins": 0.026976028457283974, |
| "rewards/rejected": -0.1634526252746582, |
| "sft_loss": 1.3647658824920654, |
| "step": 1360 |
| }, |
| { |
| "epoch": 2.436096910424539, |
| "grad_norm": 1.2747830152511597, |
| "learning_rate": 4.210510142406993e-07, |
| "logits/chosen": -20.725910186767578, |
| "logits/rejected": -20.539182662963867, |
| "logps/chosen": -1.3636181354522705, |
| "logps/rejected": -1.573249101638794, |
| "loss": 1.4312, |
| "odds_ratio_loss": 0.6754659414291382, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.13636180758476257, |
| "rewards/margins": 0.020963111892342567, |
| "rewards/rejected": -0.1573249250650406, |
| "sft_loss": 1.3636181354522705, |
| "step": 1370 |
| }, |
| { |
| "epoch": 2.4538786396977104, |
| "grad_norm": 1.5959084033966064, |
| "learning_rate": 3.9553917067232966e-07, |
| "logits/chosen": -20.685565948486328, |
| "logits/rejected": -20.612730026245117, |
| "logps/chosen": -1.3631963729858398, |
| "logps/rejected": -1.501734972000122, |
| "loss": 1.4351, |
| "odds_ratio_loss": 0.7191514372825623, |
| "rewards/accuracies": 0.5062500238418579, |
| "rewards/chosen": -0.13631963729858398, |
| "rewards/margins": 0.013853861019015312, |
| "rewards/rejected": -0.15017351508140564, |
| "sft_loss": 1.3631963729858398, |
| "step": 1380 |
| }, |
| { |
| "epoch": 2.4716603689708823, |
| "grad_norm": 1.8356739282608032, |
| "learning_rate": 3.707580696657509e-07, |
| "logits/chosen": -20.62293243408203, |
| "logits/rejected": -20.265270233154297, |
| "logps/chosen": -1.3268606662750244, |
| "logps/rejected": -1.4021425247192383, |
| "loss": 1.3996, |
| "odds_ratio_loss": 0.727665364742279, |
| "rewards/accuracies": 0.4937500059604645, |
| "rewards/chosen": -0.1326860636472702, |
| "rewards/margins": 0.007528189569711685, |
| "rewards/rejected": -0.1402142494916916, |
| "sft_loss": 1.3268606662750244, |
| "step": 1390 |
| }, |
| { |
| "epoch": 2.489442098244054, |
| "grad_norm": 2.215832471847534, |
| "learning_rate": 3.4671631608781815e-07, |
| "logits/chosen": -20.820430755615234, |
| "logits/rejected": -20.70709991455078, |
| "logps/chosen": -1.3705365657806396, |
| "logps/rejected": -1.4663138389587402, |
| "loss": 1.4448, |
| "odds_ratio_loss": 0.742554783821106, |
| "rewards/accuracies": 0.4749999940395355, |
| "rewards/chosen": -0.13705363869667053, |
| "rewards/margins": 0.009577738121151924, |
| "rewards/rejected": -0.1466313898563385, |
| "sft_loss": 1.3705365657806396, |
| "step": 1400 |
| }, |
| { |
| "epoch": 2.507223827517226, |
| "grad_norm": 2.842649221420288, |
| "learning_rate": 3.234222580780405e-07, |
| "logits/chosen": -20.579906463623047, |
| "logits/rejected": -20.50626564025879, |
| "logps/chosen": -1.3471759557724, |
| "logps/rejected": -1.4215319156646729, |
| "loss": 1.4184, |
| "odds_ratio_loss": 0.711919367313385, |
| "rewards/accuracies": 0.5249999761581421, |
| "rewards/chosen": -0.13471761345863342, |
| "rewards/margins": 0.007435592822730541, |
| "rewards/rejected": -0.14215318858623505, |
| "sft_loss": 1.3471759557724, |
| "step": 1410 |
| }, |
| { |
| "epoch": 2.525005556790398, |
| "grad_norm": 3.6692733764648438, |
| "learning_rate": 3.0088398414982375e-07, |
| "logits/chosen": -20.674327850341797, |
| "logits/rejected": -20.809429168701172, |
| "logps/chosen": -1.3552839756011963, |
| "logps/rejected": -1.5087939500808716, |
| "loss": 1.4268, |
| "odds_ratio_loss": 0.7148610353469849, |
| "rewards/accuracies": 0.543749988079071, |
| "rewards/chosen": -0.13552840054035187, |
| "rewards/margins": 0.015351003035902977, |
| "rewards/rejected": -0.1508793979883194, |
| "sft_loss": 1.3552839756011963, |
| "step": 1420 |
| }, |
| { |
| "epoch": 2.54278728606357, |
| "grad_norm": 1.8147318363189697, |
| "learning_rate": 2.7910932038184487e-07, |
| "logits/chosen": -20.291900634765625, |
| "logits/rejected": -19.921438217163086, |
| "logps/chosen": -1.3218873739242554, |
| "logps/rejected": -1.475524663925171, |
| "loss": 1.3899, |
| "odds_ratio_loss": 0.6805239319801331, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.13218875229358673, |
| "rewards/margins": 0.015363717451691628, |
| "rewards/rejected": -0.1475524604320526, |
| "sft_loss": 1.3218873739242554, |
| "step": 1430 |
| }, |
| { |
| "epoch": 2.5605690153367417, |
| "grad_norm": 2.6163878440856934, |
| "learning_rate": 2.5810582770057325e-07, |
| "logits/chosen": -20.752613067626953, |
| "logits/rejected": -20.92694854736328, |
| "logps/chosen": -1.2694684267044067, |
| "logps/rejected": -1.3633973598480225, |
| "loss": 1.3412, |
| "odds_ratio_loss": 0.7170311212539673, |
| "rewards/accuracies": 0.5062500238418579, |
| "rewards/chosen": -0.1269468367099762, |
| "rewards/margins": 0.009392908774316311, |
| "rewards/rejected": -0.13633975386619568, |
| "sft_loss": 1.2694684267044067, |
| "step": 1440 |
| }, |
| { |
| "epoch": 2.578350744609913, |
| "grad_norm": 2.4267303943634033, |
| "learning_rate": 2.3788079925484402e-07, |
| "logits/chosen": -20.907817840576172, |
| "logits/rejected": -20.742984771728516, |
| "logps/chosen": -1.3328653573989868, |
| "logps/rejected": -1.4286470413208008, |
| "loss": 1.4041, |
| "odds_ratio_loss": 0.7125917673110962, |
| "rewards/accuracies": 0.512499988079071, |
| "rewards/chosen": -0.1332865208387375, |
| "rewards/margins": 0.009578163735568523, |
| "rewards/rejected": -0.14286470413208008, |
| "sft_loss": 1.3328653573989868, |
| "step": 1450 |
| }, |
| { |
| "epoch": 2.596132473883085, |
| "grad_norm": 2.563065528869629, |
| "learning_rate": 2.1844125788342661e-07, |
| "logits/chosen": -20.36819076538086, |
| "logits/rejected": -20.245798110961914, |
| "logps/chosen": -1.3011656999588013, |
| "logps/rejected": -1.601665735244751, |
| "loss": 1.3692, |
| "odds_ratio_loss": 0.679993212223053, |
| "rewards/accuracies": 0.6000000238418579, |
| "rewards/chosen": -0.1301165670156479, |
| "rewards/margins": 0.030050003901124, |
| "rewards/rejected": -0.16016657650470734, |
| "sft_loss": 1.3011656999588013, |
| "step": 1460 |
| }, |
| { |
| "epoch": 2.613914203156257, |
| "grad_norm": 1.592044711112976, |
| "learning_rate": 1.9979395367644428e-07, |
| "logits/chosen": -20.988916397094727, |
| "logits/rejected": -20.9386043548584, |
| "logps/chosen": -1.2825675010681152, |
| "logps/rejected": -1.444154977798462, |
| "loss": 1.3506, |
| "odds_ratio_loss": 0.6804038286209106, |
| "rewards/accuracies": 0.5249999761581421, |
| "rewards/chosen": -0.12825676798820496, |
| "rewards/margins": 0.01615874283015728, |
| "rewards/rejected": -0.1444154977798462, |
| "sft_loss": 1.2825675010681152, |
| "step": 1470 |
| }, |
| { |
| "epoch": 2.631695932429429, |
| "grad_norm": 3.1699938774108887, |
| "learning_rate": 1.81945361631512e-07, |
| "logits/chosen": -21.14181900024414, |
| "logits/rejected": -21.21465301513672, |
| "logps/chosen": -1.3464009761810303, |
| "logps/rejected": -1.4395297765731812, |
| "loss": 1.4199, |
| "odds_ratio_loss": 0.7345655560493469, |
| "rewards/accuracies": 0.4937500059604645, |
| "rewards/chosen": -0.13464009761810303, |
| "rewards/margins": 0.00931286346167326, |
| "rewards/rejected": -0.14395298063755035, |
| "sft_loss": 1.3464009761810303, |
| "step": 1480 |
| }, |
| { |
| "epoch": 2.6494776617026004, |
| "grad_norm": 2.9426472187042236, |
| "learning_rate": 1.6490167940538343e-07, |
| "logits/chosen": -20.980464935302734, |
| "logits/rejected": -20.800823211669922, |
| "logps/chosen": -1.3066675662994385, |
| "logps/rejected": -1.4621120691299438, |
| "loss": 1.3758, |
| "odds_ratio_loss": 0.691235363483429, |
| "rewards/accuracies": 0.5062500238418579, |
| "rewards/chosen": -0.13066676259040833, |
| "rewards/margins": 0.015544441528618336, |
| "rewards/rejected": -0.1462111920118332, |
| "sft_loss": 1.3066675662994385, |
| "step": 1490 |
| }, |
| { |
| "epoch": 2.6672593909757722, |
| "grad_norm": 2.0618135929107666, |
| "learning_rate": 1.4866882516191339e-07, |
| "logits/chosen": -20.438335418701172, |
| "logits/rejected": -20.611011505126953, |
| "logps/chosen": -1.3370510339736938, |
| "logps/rejected": -1.4072545766830444, |
| "loss": 1.4105, |
| "odds_ratio_loss": 0.7341033220291138, |
| "rewards/accuracies": 0.5249999761581421, |
| "rewards/chosen": -0.13370510935783386, |
| "rewards/margins": 0.007020360324531794, |
| "rewards/rejected": -0.14072546362876892, |
| "sft_loss": 1.3370510339736938, |
| "step": 1500 |
| }, |
| { |
| "epoch": 2.6672593909757722, |
| "eval_logits/chosen": -20.655466079711914, |
| "eval_logits/rejected": -20.744272232055664, |
| "eval_logps/chosen": -1.3319367170333862, |
| "eval_logps/rejected": -1.5009632110595703, |
| "eval_loss": 1.3996269702911377, |
| "eval_odds_ratio_loss": 0.67690110206604, |
| "eval_rewards/accuracies": 0.5460000038146973, |
| "eval_rewards/chosen": -0.133193701505661, |
| "eval_rewards/margins": 0.016902634873986244, |
| "eval_rewards/rejected": -0.1500963419675827, |
| "eval_runtime": 80.0771, |
| "eval_samples_per_second": 12.488, |
| "eval_sft_loss": 1.3319367170333862, |
| "eval_steps_per_second": 6.244, |
| "step": 1500 |
| }, |
| { |
| "epoch": 2.685041120248944, |
| "grad_norm": 4.3807573318481445, |
| "learning_rate": 1.3325243551706057e-07, |
| "logits/chosen": -20.31595802307129, |
| "logits/rejected": -20.66552734375, |
| "logps/chosen": -1.3314330577850342, |
| "logps/rejected": -1.618486762046814, |
| "loss": 1.3972, |
| "odds_ratio_loss": 0.6577640175819397, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.13314330577850342, |
| "rewards/margins": 0.028705382719635963, |
| "rewards/rejected": -0.16184869408607483, |
| "sft_loss": 1.3314330577850342, |
| "step": 1510 |
| }, |
| { |
| "epoch": 2.702822849522116, |
| "grad_norm": 5.399725437164307, |
| "learning_rate": 1.1865786358165737e-07, |
| "logits/chosen": -20.374225616455078, |
| "logits/rejected": -20.766555786132812, |
| "logps/chosen": -1.3559472560882568, |
| "logps/rejected": -1.465380311012268, |
| "loss": 1.4273, |
| "odds_ratio_loss": 0.71399986743927, |
| "rewards/accuracies": 0.518750011920929, |
| "rewards/chosen": -0.13559472560882568, |
| "rewards/margins": 0.010943309403955936, |
| "rewards/rejected": -0.14653804898262024, |
| "sft_loss": 1.3559472560882568, |
| "step": 1520 |
| }, |
| { |
| "epoch": 2.720604578795288, |
| "grad_norm": 5.161293029785156, |
| "learning_rate": 1.0489017710262311e-07, |
| "logits/chosen": -20.828411102294922, |
| "logits/rejected": -21.007801055908203, |
| "logps/chosen": -1.3781417608261108, |
| "logps/rejected": -1.6057850122451782, |
| "loss": 1.4502, |
| "odds_ratio_loss": 0.7206953763961792, |
| "rewards/accuracies": 0.574999988079071, |
| "rewards/chosen": -0.13781419396400452, |
| "rewards/margins": 0.02276432514190674, |
| "rewards/rejected": -0.16057850420475006, |
| "sft_loss": 1.3781417608261108, |
| "step": 1530 |
| }, |
| { |
| "epoch": 2.73838630806846, |
| "grad_norm": 1.9645308256149292, |
| "learning_rate": 9.195415670326446e-08, |
| "logits/chosen": -20.687061309814453, |
| "logits/rejected": -20.75905990600586, |
| "logps/chosen": -1.3485379219055176, |
| "logps/rejected": -1.5022733211517334, |
| "loss": 1.4177, |
| "odds_ratio_loss": 0.6911865472793579, |
| "rewards/accuracies": 0.512499988079071, |
| "rewards/chosen": -0.1348538100719452, |
| "rewards/margins": 0.015373537316918373, |
| "rewards/rejected": -0.150227352976799, |
| "sft_loss": 1.3485379219055176, |
| "step": 1540 |
| }, |
| { |
| "epoch": 2.7561680373416317, |
| "grad_norm": 2.820127010345459, |
| "learning_rate": 7.985429422327384e-08, |
| "logits/chosen": -20.722209930419922, |
| "logits/rejected": -20.719024658203125, |
| "logps/chosen": -1.3103783130645752, |
| "logps/rejected": -1.3722031116485596, |
| "loss": 1.3834, |
| "odds_ratio_loss": 0.7300290465354919, |
| "rewards/accuracies": 0.48750001192092896, |
| "rewards/chosen": -0.1310378611087799, |
| "rewards/margins": 0.006182484794408083, |
| "rewards/rejected": -0.1372203379869461, |
| "sft_loss": 1.3103783130645752, |
| "step": 1550 |
| }, |
| { |
| "epoch": 2.773949766614803, |
| "grad_norm": 3.8620612621307373, |
| "learning_rate": 6.859479115900818e-08, |
| "logits/chosen": -20.64493179321289, |
| "logits/rejected": -20.700607299804688, |
| "logps/chosen": -1.4513204097747803, |
| "logps/rejected": -1.611519455909729, |
| "loss": 1.5216, |
| "odds_ratio_loss": 0.7024157047271729, |
| "rewards/accuracies": 0.518750011920929, |
| "rewards/chosen": -0.14513204991817474, |
| "rewards/margins": 0.01601991057395935, |
| "rewards/rejected": -0.1611519604921341, |
| "sft_loss": 1.4513204097747803, |
| "step": 1560 |
| }, |
| { |
| "epoch": 2.791731495887975, |
| "grad_norm": 8.068270683288574, |
| "learning_rate": 5.817955720457902e-08, |
| "logits/chosen": -20.495128631591797, |
| "logits/rejected": -20.559017181396484, |
| "logps/chosen": -1.27366042137146, |
| "logps/rejected": -1.3641878366470337, |
| "loss": 1.3449, |
| "odds_ratio_loss": 0.7120680809020996, |
| "rewards/accuracies": 0.543749988079071, |
| "rewards/chosen": -0.12736603617668152, |
| "rewards/margins": 0.009052738547325134, |
| "rewards/rejected": -0.13641878962516785, |
| "sft_loss": 1.27366042137146, |
| "step": 1570 |
| }, |
| { |
| "epoch": 2.809513225161147, |
| "grad_norm": 3.788001775741577, |
| "learning_rate": 4.861220889427199e-08, |
| "logits/chosen": -20.778738021850586, |
| "logits/rejected": -20.58936309814453, |
| "logps/chosen": -1.3479427099227905, |
| "logps/rejected": -1.4240316152572632, |
| "loss": 1.423, |
| "odds_ratio_loss": 0.7501288652420044, |
| "rewards/accuracies": 0.543749988079071, |
| "rewards/chosen": -0.13479426503181458, |
| "rewards/margins": 0.007608892861753702, |
| "rewards/rejected": -0.14240317046642303, |
| "sft_loss": 1.3479427099227905, |
| "step": 1580 |
| }, |
| { |
| "epoch": 2.827294954434319, |
| "grad_norm": 1.3882092237472534, |
| "learning_rate": 3.9896068346758074e-08, |
| "logits/chosen": -20.645978927612305, |
| "logits/rejected": -20.691020965576172, |
| "logps/chosen": -1.3436458110809326, |
| "logps/rejected": -1.4700592756271362, |
| "loss": 1.4137, |
| "odds_ratio_loss": 0.7008241415023804, |
| "rewards/accuracies": 0.518750011920929, |
| "rewards/chosen": -0.13436457514762878, |
| "rewards/margins": 0.012641333043575287, |
| "rewards/rejected": -0.14700593054294586, |
| "sft_loss": 1.3436458110809326, |
| "step": 1590 |
| }, |
| { |
| "epoch": 2.8450766837074903, |
| "grad_norm": 3.9510364532470703, |
| "learning_rate": 3.203416211153832e-08, |
| "logits/chosen": -20.51412582397461, |
| "logits/rejected": -20.81777572631836, |
| "logps/chosen": -1.331721544265747, |
| "logps/rejected": -1.4455146789550781, |
| "loss": 1.4049, |
| "odds_ratio_loss": 0.7322754859924316, |
| "rewards/accuracies": 0.4625000059604645, |
| "rewards/chosen": -0.1331721693277359, |
| "rewards/margins": 0.011379324831068516, |
| "rewards/rejected": -0.14455147087574005, |
| "sft_loss": 1.331721544265747, |
| "step": 1600 |
| }, |
| { |
| "epoch": 2.8628584129806622, |
| "grad_norm": 5.2995758056640625, |
| "learning_rate": 2.5029220118019393e-08, |
| "logits/chosen": -20.452526092529297, |
| "logits/rejected": -20.59510612487793, |
| "logps/chosen": -1.3901276588439941, |
| "logps/rejected": -1.4555182456970215, |
| "loss": 1.4637, |
| "odds_ratio_loss": 0.7356002330780029, |
| "rewards/accuracies": 0.48750001192092896, |
| "rewards/chosen": -0.13901275396347046, |
| "rewards/margins": 0.006539070047438145, |
| "rewards/rejected": -0.14555183053016663, |
| "sft_loss": 1.3901276588439941, |
| "step": 1610 |
| }, |
| { |
| "epoch": 2.880640142253834, |
| "grad_norm": 2.9100406169891357, |
| "learning_rate": 1.8883674727586122e-08, |
| "logits/chosen": -20.593090057373047, |
| "logits/rejected": -20.566762924194336, |
| "logps/chosen": -1.2591346502304077, |
| "logps/rejected": -1.5285673141479492, |
| "loss": 1.3231, |
| "odds_ratio_loss": 0.6392361521720886, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.12591347098350525, |
| "rewards/margins": 0.026943260803818703, |
| "rewards/rejected": -0.1528567224740982, |
| "sft_loss": 1.2591346502304077, |
| "step": 1620 |
| }, |
| { |
| "epoch": 2.898421871527006, |
| "grad_norm": 1.2503418922424316, |
| "learning_rate": 1.3599659889000639e-08, |
| "logits/chosen": -20.94070816040039, |
| "logits/rejected": -20.831439971923828, |
| "logps/chosen": -1.3583745956420898, |
| "logps/rejected": -1.4623037576675415, |
| "loss": 1.4301, |
| "odds_ratio_loss": 0.7174537181854248, |
| "rewards/accuracies": 0.48124998807907104, |
| "rewards/chosen": -0.13583745062351227, |
| "rewards/margins": 0.010392926633358002, |
| "rewards/rejected": -0.14623036980628967, |
| "sft_loss": 1.3583745956420898, |
| "step": 1630 |
| }, |
| { |
| "epoch": 2.916203600800178, |
| "grad_norm": 1.5849334001541138, |
| "learning_rate": 9.179010397421528e-09, |
| "logits/chosen": -20.463802337646484, |
| "logits/rejected": -20.60258674621582, |
| "logps/chosen": -1.3220821619033813, |
| "logps/rejected": -1.4576328992843628, |
| "loss": 1.3923, |
| "odds_ratio_loss": 0.7020986676216125, |
| "rewards/accuracies": 0.4749999940395355, |
| "rewards/chosen": -0.1322081983089447, |
| "rewards/margins": 0.01355508528649807, |
| "rewards/rejected": -0.1457633078098297, |
| "sft_loss": 1.3220821619033813, |
| "step": 1640 |
| }, |
| { |
| "epoch": 2.93398533007335, |
| "grad_norm": 1.5637987852096558, |
| "learning_rate": 5.623261257296509e-09, |
| "logits/chosen": -20.550914764404297, |
| "logits/rejected": -20.748790740966797, |
| "logps/chosen": -1.2091234922409058, |
| "logps/rejected": -1.3757600784301758, |
| "loss": 1.2759, |
| "odds_ratio_loss": 0.6680835485458374, |
| "rewards/accuracies": 0.512499988079071, |
| "rewards/chosen": -0.1209123507142067, |
| "rewards/margins": 0.01666366681456566, |
| "rewards/rejected": -0.13757601380348206, |
| "sft_loss": 1.2091234922409058, |
| "step": 1650 |
| }, |
| { |
| "epoch": 2.9517670593465217, |
| "grad_norm": 2.060124158859253, |
| "learning_rate": 2.933647149357122e-09, |
| "logits/chosen": -20.7076358795166, |
| "logits/rejected": -20.74884605407715, |
| "logps/chosen": -1.344455361366272, |
| "logps/rejected": -1.4792556762695312, |
| "loss": 1.415, |
| "odds_ratio_loss": 0.7057270407676697, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.13444553315639496, |
| "rewards/margins": 0.013480030000209808, |
| "rewards/rejected": -0.14792557060718536, |
| "sft_loss": 1.344455361366272, |
| "step": 1660 |
| }, |
| { |
| "epoch": 2.969548788619693, |
| "grad_norm": 1.8397283554077148, |
| "learning_rate": 1.1111020018930717e-09, |
| "logits/chosen": -20.895946502685547, |
| "logits/rejected": -20.76508903503418, |
| "logps/chosen": -1.319896936416626, |
| "logps/rejected": -1.4024416208267212, |
| "loss": 1.3912, |
| "odds_ratio_loss": 0.7127953767776489, |
| "rewards/accuracies": 0.48750001192092896, |
| "rewards/chosen": -0.13198968768119812, |
| "rewards/margins": 0.008254442363977432, |
| "rewards/rejected": -0.14024415612220764, |
| "sft_loss": 1.319896936416626, |
| "step": 1670 |
| }, |
| { |
| "epoch": 2.987330517892865, |
| "grad_norm": 1.2750743627548218, |
| "learning_rate": 1.5625866646051813e-10, |
| "logits/chosen": -20.67104721069336, |
| "logits/rejected": -20.601062774658203, |
| "logps/chosen": -1.2744053602218628, |
| "logps/rejected": -1.4778211116790771, |
| "loss": 1.3388, |
| "odds_ratio_loss": 0.6436463594436646, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.12744054198265076, |
| "rewards/margins": 0.02034156210720539, |
| "rewards/rejected": -0.1477821171283722, |
| "sft_loss": 1.2744053602218628, |
| "step": 1680 |
| }, |
| { |
| "epoch": 2.997999555456768, |
| "step": 1686, |
| "total_flos": 5.313908590588723e+17, |
| "train_loss": 1.477055920117832, |
| "train_runtime": 8055.1491, |
| "train_samples_per_second": 3.351, |
| "train_steps_per_second": 0.209 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 1686, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 3, |
| "save_steps": 500, |
| "total_flos": 5.313908590588723e+17, |
| "train_batch_size": 2, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
