| { |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.9968652037617555, |
| "eval_steps": 500, |
| "global_step": 159, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.01, |
| "learning_rate": 3.125e-08, |
| "logits/chosen": -2.6344056129455566, |
| "logits/rejected": -2.5906338691711426, |
| "logps/chosen": -158.64126586914062, |
| "logps/pi_response": -86.09246826171875, |
| "logps/ref_response": -86.09246826171875, |
| "logps/rejected": -148.42047119140625, |
| "loss": 0.6931, |
| "rewards/accuracies": 0.0, |
| "rewards/chosen": 0.0, |
| "rewards/margins": 0.0, |
| "rewards/rejected": 0.0, |
| "step": 1 |
| }, |
| { |
| "epoch": 0.06, |
| "learning_rate": 3.1249999999999997e-07, |
| "logits/chosen": -2.6920104026794434, |
| "logits/rejected": -2.6961071491241455, |
| "logps/chosen": -233.56158447265625, |
| "logps/pi_response": -119.84620666503906, |
| "logps/ref_response": -118.89913940429688, |
| "logps/rejected": -252.62852478027344, |
| "loss": 0.692, |
| "rewards/accuracies": 0.4444444477558136, |
| "rewards/chosen": -0.01218863669782877, |
| "rewards/margins": 0.0017457219073548913, |
| "rewards/rejected": -0.013934357091784477, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.13, |
| "learning_rate": 4.990353313429303e-07, |
| "logits/chosen": -2.6988842487335205, |
| "logits/rejected": -2.670401096343994, |
| "logps/chosen": -249.9190216064453, |
| "logps/pi_response": -146.00906372070312, |
| "logps/ref_response": -131.12376403808594, |
| "logps/rejected": -276.4181823730469, |
| "loss": 0.6796, |
| "rewards/accuracies": 0.5687500238418579, |
| "rewards/chosen": -0.17617936432361603, |
| "rewards/margins": 0.050668977200984955, |
| "rewards/rejected": -0.2268483191728592, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.19, |
| "learning_rate": 4.882681251368548e-07, |
| "logits/chosen": -2.5072598457336426, |
| "logits/rejected": -2.4469008445739746, |
| "logps/chosen": -241.61929321289062, |
| "logps/pi_response": -130.57382202148438, |
| "logps/ref_response": -113.8835220336914, |
| "logps/rejected": -247.4310760498047, |
| "loss": 0.6664, |
| "rewards/accuracies": 0.5687500238418579, |
| "rewards/chosen": -0.15306231379508972, |
| "rewards/margins": 0.11979613453149796, |
| "rewards/rejected": -0.2728584408760071, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.25, |
| "learning_rate": 4.6604720940421207e-07, |
| "logits/chosen": -2.322709798812866, |
| "logits/rejected": -2.300786256790161, |
| "logps/chosen": -245.057373046875, |
| "logps/pi_response": -155.58706665039062, |
| "logps/ref_response": -110.53662109375, |
| "logps/rejected": -320.01849365234375, |
| "loss": 0.6421, |
| "rewards/accuracies": 0.6937500238418579, |
| "rewards/chosen": -0.4277011454105377, |
| "rewards/margins": 0.18936362862586975, |
| "rewards/rejected": -0.6170647740364075, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.31, |
| "learning_rate": 4.3344075855595097e-07, |
| "logits/chosen": -2.2339208126068115, |
| "logits/rejected": -2.2038655281066895, |
| "logps/chosen": -281.0859375, |
| "logps/pi_response": -177.91244506835938, |
| "logps/ref_response": -120.1254653930664, |
| "logps/rejected": -306.99078369140625, |
| "loss": 0.6311, |
| "rewards/accuracies": 0.643750011920929, |
| "rewards/chosen": -0.552875280380249, |
| "rewards/margins": 0.14258158206939697, |
| "rewards/rejected": -0.695456862449646, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.38, |
| "learning_rate": 3.920161866827889e-07, |
| "logits/chosen": -2.136162281036377, |
| "logits/rejected": -2.102271556854248, |
| "logps/chosen": -285.7301330566406, |
| "logps/pi_response": -197.15200805664062, |
| "logps/ref_response": -107.5267333984375, |
| "logps/rejected": -330.02581787109375, |
| "loss": 0.632, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": -0.7207737565040588, |
| "rewards/margins": 0.32419511675834656, |
| "rewards/rejected": -1.044968843460083, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.44, |
| "learning_rate": 3.4376480090239047e-07, |
| "logits/chosen": -2.116626024246216, |
| "logits/rejected": -2.0622029304504395, |
| "logps/chosen": -290.5029602050781, |
| "logps/pi_response": -204.41468811035156, |
| "logps/ref_response": -112.31196594238281, |
| "logps/rejected": -308.43865966796875, |
| "loss": 0.6112, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.7230253219604492, |
| "rewards/margins": 0.3711085617542267, |
| "rewards/rejected": -1.0941338539123535, |
| "step": 70 |
| }, |
| { |
| "epoch": 0.5, |
| "learning_rate": 2.910060778827554e-07, |
| "logits/chosen": -2.1222305297851562, |
| "logits/rejected": -2.0609357357025146, |
| "logps/chosen": -270.329345703125, |
| "logps/pi_response": -193.92774963378906, |
| "logps/ref_response": -99.59819793701172, |
| "logps/rejected": -328.5818176269531, |
| "loss": 0.6234, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.7498321533203125, |
| "rewards/margins": 0.32974669337272644, |
| "rewards/rejected": -1.0795788764953613, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.56, |
| "learning_rate": 2.3627616503391812e-07, |
| "logits/chosen": -2.1789047718048096, |
| "logits/rejected": -2.1352646350860596, |
| "logps/chosen": -281.3520202636719, |
| "logps/pi_response": -184.90760803222656, |
| "logps/ref_response": -100.86174011230469, |
| "logps/rejected": -323.51318359375, |
| "loss": 0.6141, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -0.7342792749404907, |
| "rewards/margins": 0.34946689009666443, |
| "rewards/rejected": -1.0837461948394775, |
| "step": 90 |
| }, |
| { |
| "epoch": 0.63, |
| "learning_rate": 1.8220596619089573e-07, |
| "logits/chosen": -2.075347661972046, |
| "logits/rejected": -2.0635223388671875, |
| "logps/chosen": -291.8459167480469, |
| "logps/pi_response": -209.01785278320312, |
| "logps/ref_response": -112.6884765625, |
| "logps/rejected": -376.8053894042969, |
| "loss": 0.5877, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -0.8351489305496216, |
| "rewards/margins": 0.3953370749950409, |
| "rewards/rejected": -1.2304860353469849, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.69, |
| "learning_rate": 1.3139467229135998e-07, |
| "logits/chosen": -2.1212668418884277, |
| "logits/rejected": -2.1371259689331055, |
| "logps/chosen": -294.1075439453125, |
| "logps/pi_response": -189.68203735351562, |
| "logps/ref_response": -100.60249328613281, |
| "logps/rejected": -319.2774963378906, |
| "loss": 0.6056, |
| "rewards/accuracies": 0.606249988079071, |
| "rewards/chosen": -0.7740401029586792, |
| "rewards/margins": 0.3121982216835022, |
| "rewards/rejected": -1.086238145828247, |
| "step": 110 |
| }, |
| { |
| "epoch": 0.75, |
| "learning_rate": 8.628481651367875e-08, |
| "logits/chosen": -2.127702236175537, |
| "logits/rejected": -2.052624225616455, |
| "logps/chosen": -323.14556884765625, |
| "logps/pi_response": -227.27230834960938, |
| "logps/ref_response": -130.39340209960938, |
| "logps/rejected": -348.0731506347656, |
| "loss": 0.5969, |
| "rewards/accuracies": 0.6187499761581421, |
| "rewards/chosen": -0.8538217544555664, |
| "rewards/margins": 0.24452969431877136, |
| "rewards/rejected": -1.0983514785766602, |
| "step": 120 |
| }, |
| { |
| "epoch": 0.82, |
| "learning_rate": 4.904486005914027e-08, |
| "logits/chosen": -2.16937518119812, |
| "logits/rejected": -2.149695634841919, |
| "logps/chosen": -293.9873046875, |
| "logps/pi_response": -207.9149932861328, |
| "logps/ref_response": -113.8158950805664, |
| "logps/rejected": -352.2449035644531, |
| "loss": 0.5837, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -0.8029237985610962, |
| "rewards/margins": 0.38218849897384644, |
| "rewards/rejected": -1.1851122379302979, |
| "step": 130 |
| }, |
| { |
| "epoch": 0.88, |
| "learning_rate": 2.1464952759020856e-08, |
| "logits/chosen": -2.166949987411499, |
| "logits/rejected": -2.0973129272460938, |
| "logps/chosen": -321.9905090332031, |
| "logps/pi_response": -238.9150390625, |
| "logps/ref_response": -119.49796295166016, |
| "logps/rejected": -366.47198486328125, |
| "loss": 0.5936, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.9360674023628235, |
| "rewards/margins": 0.3974238932132721, |
| "rewards/rejected": -1.3334912061691284, |
| "step": 140 |
| }, |
| { |
| "epoch": 0.94, |
| "learning_rate": 4.8708793644441086e-09, |
| "logits/chosen": -2.112703800201416, |
| "logits/rejected": -2.0817809104919434, |
| "logps/chosen": -298.79486083984375, |
| "logps/pi_response": -201.3252716064453, |
| "logps/ref_response": -101.72931671142578, |
| "logps/rejected": -328.84320068359375, |
| "loss": 0.5747, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -0.8357221484184265, |
| "rewards/margins": 0.33179959654808044, |
| "rewards/rejected": -1.1675217151641846, |
| "step": 150 |
| }, |
| { |
| "epoch": 1.0, |
| "step": 159, |
| "total_flos": 0.0, |
| "train_loss": 0.6203871613028664, |
| "train_runtime": 4552.526, |
| "train_samples_per_second": 4.476, |
| "train_steps_per_second": 0.035 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 159, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 100, |
| "total_flos": 0.0, |
| "train_batch_size": 8, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
