{ "best_metric": null, "best_model_checkpoint": null, "epoch": 1.0, "eval_steps": 10, "global_step": 1500, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio": 0.0, "completion_length": 514.2500152587891, "epoch": 0.0006666666666666666, "grad_norm": 0.21932029724121094, "kl": 0.0, "learning_rate": 2e-08, "loss": 0.091, "reward": 0.0833333358168602, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 1 }, { "clip_ratio": 0.0, "completion_length": 741.4583435058594, "epoch": 0.0013333333333333333, "grad_norm": 0.0, "kl": 0.0, "learning_rate": 4e-08, "loss": 0.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 2 }, { "clip_ratio": 0.0, "completion_length": 532.5833549499512, "epoch": 0.002, "grad_norm": 0.0016497765900567174, "kl": 0.00020003318786621094, "learning_rate": 6.000000000000001e-08, "loss": 0.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 3 }, { "clip_ratio": 0.0, "completion_length": 561.1250076293945, "epoch": 0.0026666666666666666, "grad_norm": 0.18049472570419312, "kl": 0.0002206563949584961, "learning_rate": 8e-08, "loss": 0.1113, "reward": 0.0833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 4 }, { "clip_ratio": 0.0, "completion_length": 544.3333435058594, "epoch": 0.0033333333333333335, "grad_norm": 0.15046288073062897, "kl": 0.00022673606872558594, "learning_rate": 1e-07, "loss": -0.0639, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 5 }, { "clip_ratio": 0.0, "completion_length": 554.9166946411133, "epoch": 0.004, "grad_norm": 0.21441583335399628, "kl": 0.0001690387725830078, "learning_rate": 1.2000000000000002e-07, "loss": -0.0505, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 6 }, { "clip_ratio": 0.0, "completion_length": 391.87500762939453, "epoch": 0.004666666666666667, "grad_norm": 0.32293665409088135, "kl": 0.00022995471954345703, "learning_rate": 1.4e-07, "loss": -0.0615, "reward": 0.1250000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 7 }, { "clip_ratio": 0.0, "completion_length": 538.2500152587891, "epoch": 0.005333333333333333, "grad_norm": 0.48185980319976807, "kl": 0.00017631053924560547, "learning_rate": 1.6e-07, "loss": 0.0016, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 8 }, { "clip_ratio": 0.0, "completion_length": 569.6666717529297, "epoch": 0.006, "grad_norm": 0.0014477341901510954, "kl": 0.00020503997802734375, "learning_rate": 1.8e-07, "loss": 0.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 9 }, { "clip_ratio": 0.0, "completion_length": 615.4583511352539, "epoch": 0.006666666666666667, "grad_norm": 0.0015941731398925185, "kl": 0.0001850128173828125, "learning_rate": 2e-07, "loss": 0.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 10 }, { "clip_ratio": 0.0, "completion_length": 521.0000076293945, "epoch": 0.007333333333333333, "grad_norm": 0.22514379024505615, "kl": 0.0002282857894897461, "learning_rate": 2.2e-07, "loss": 0.0033, "reward": 0.0833333358168602, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 11 }, { "clip_ratio": 0.0, "completion_length": 582.0833511352539, "epoch": 0.008, "grad_norm": 0.0017629085341468453, "kl": 0.00012451410293579102, "learning_rate": 2.4000000000000003e-07, "loss": 0.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 12 }, { "clip_ratio": 0.0, "completion_length": 497.87500762939453, "epoch": 0.008666666666666666, "grad_norm": 0.8173198103904724, "kl": 0.00021576881408691406, "learning_rate": 2.6e-07, "loss": 0.0146, "reward": 0.1250000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 13 }, { "clip_ratio": 0.0, "completion_length": 546.0416870117188, "epoch": 0.009333333333333334, "grad_norm": 0.28712546825408936, "kl": 0.0002734661102294922, "learning_rate": 2.8e-07, "loss": -0.031, "reward": 0.0833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 14 }, { "clip_ratio": 0.0, "completion_length": 425.4583435058594, "epoch": 0.01, "grad_norm": 0.22834625840187073, "kl": 0.0001531839370727539, "learning_rate": 3.0000000000000004e-07, "loss": 0.0124, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 15 }, { "clip_ratio": 0.0, "completion_length": 517.3333511352539, "epoch": 0.010666666666666666, "grad_norm": 0.3629693388938904, "kl": 0.00016963481903076172, "learning_rate": 3.2e-07, "loss": -0.0272, "reward": 0.1250000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 16 }, { "clip_ratio": 0.0, "completion_length": 736.7916717529297, "epoch": 0.011333333333333334, "grad_norm": 0.27569979429244995, "kl": 0.0002427995204925537, "learning_rate": 3.4e-07, "loss": 0.0897, "reward": 0.1666666679084301, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 17 }, { "clip_ratio": 0.0, "completion_length": 593.5833587646484, "epoch": 0.012, "grad_norm": 0.4576092064380646, "kl": 0.00021207332611083984, "learning_rate": 3.6e-07, "loss": -0.087, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 18 }, { "clip_ratio": 0.0, "completion_length": 431.0416889190674, "epoch": 0.012666666666666666, "grad_norm": 0.4593261182308197, "kl": 0.00025844573974609375, "learning_rate": 3.8000000000000007e-07, "loss": 0.074, "reward": 0.1666666716337204, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 19 }, { "clip_ratio": 0.0, "completion_length": 392.0416717529297, "epoch": 0.013333333333333334, "grad_norm": 0.2665972411632538, "kl": 0.00028252601623535156, "learning_rate": 4e-07, "loss": -0.0475, "reward": 0.0416666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 20 }, { "clip_ratio": 0.0, "completion_length": 448.1666793823242, "epoch": 0.014, "grad_norm": 0.0021293966565281153, "kl": 0.00024366378784179688, "learning_rate": 4.2000000000000006e-07, "loss": 0.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 21 }, { "clip_ratio": 0.0, "completion_length": 616.2916793823242, "epoch": 0.014666666666666666, "grad_norm": 0.14931456744670868, "kl": 0.0001938343048095703, "learning_rate": 4.4e-07, "loss": 0.0846, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 22 }, { "clip_ratio": 0.0, "completion_length": 601.4583587646484, "epoch": 0.015333333333333332, "grad_norm": 0.09677352756261826, "kl": 0.00021541118621826172, "learning_rate": 4.6e-07, "loss": 0.0, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 23 }, { "clip_ratio": 0.0, "completion_length": 461.3333511352539, "epoch": 0.016, "grad_norm": 0.09333957731723785, "kl": 0.0002148151397705078, "learning_rate": 4.800000000000001e-07, "loss": 0.0506, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 24 }, { "clip_ratio": 0.0, "completion_length": 620.6666717529297, "epoch": 0.016666666666666666, "grad_norm": 0.10496797412633896, "kl": 0.00017786026000976562, "learning_rate": 5e-07, "loss": 0.0523, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 25 }, { "clip_ratio": 0.0, "completion_length": 541.2500152587891, "epoch": 0.017333333333333333, "grad_norm": 0.12531428039073944, "kl": 0.00014072656631469727, "learning_rate": 5.2e-07, "loss": 0.0718, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 26 }, { "clip_ratio": 0.0, "completion_length": 506.8333511352539, "epoch": 0.018, "grad_norm": 0.20736518502235413, "kl": 0.00034499168395996094, "learning_rate": 5.4e-07, "loss": -0.0395, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 27 }, { "clip_ratio": 0.0, "completion_length": 576.4583435058594, "epoch": 0.018666666666666668, "grad_norm": 0.2102038413286209, "kl": 0.0003094673156738281, "learning_rate": 5.6e-07, "loss": -0.0607, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 28 }, { "clip_ratio": 0.0, "completion_length": 619.7500228881836, "epoch": 0.019333333333333334, "grad_norm": 0.2700510323047638, "kl": 0.00025200843811035156, "learning_rate": 5.8e-07, "loss": -0.062, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 29 }, { "clip_ratio": 0.0, "completion_length": 723.2916870117188, "epoch": 0.02, "grad_norm": 0.1831643283367157, "kl": 0.00018978118896484375, "learning_rate": 6.000000000000001e-07, "loss": -0.0311, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 30 }, { "clip_ratio": 0.0, "completion_length": 608.3750228881836, "epoch": 0.020666666666666667, "grad_norm": 0.26690390706062317, "kl": 0.00031828880310058594, "learning_rate": 6.2e-07, "loss": 0.0204, "reward": 0.1666666716337204, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 31 }, { "clip_ratio": 0.0, "completion_length": 523.5833511352539, "epoch": 0.021333333333333333, "grad_norm": 0.0013482890790328383, "kl": 0.00033402442932128906, "learning_rate": 6.4e-07, "loss": 0.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 32 }, { "clip_ratio": 0.0, "completion_length": 563.5416793823242, "epoch": 0.022, "grad_norm": 0.0013153532054275274, "kl": 0.00019502639770507812, "learning_rate": 6.6e-07, "loss": 0.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 33 }, { "clip_ratio": 0.0, "completion_length": 416.2916717529297, "epoch": 0.02266666666666667, "grad_norm": 0.34072959423065186, "kl": 0.0002834796905517578, "learning_rate": 6.8e-07, "loss": 0.0501, "reward": 0.1250000037252903, "reward_std": 0.3061862140893936, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 34 }, { "clip_ratio": 0.0, "completion_length": 581.9583511352539, "epoch": 0.023333333333333334, "grad_norm": 0.5107926726341248, "kl": 0.0001919269561767578, "learning_rate": 7.000000000000001e-07, "loss": 0.0, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 35 }, { "clip_ratio": 0.0, "completion_length": 454.6250114440918, "epoch": 0.024, "grad_norm": 0.3955595791339874, "kl": 0.00029778480529785156, "learning_rate": 7.2e-07, "loss": -0.1396, "reward": 0.2916666679084301, "reward_std": 0.37592336535453796, "rewards/accuracy_reward": 0.2916666679084301, "rewards/format_reward": 0.0, "step": 36 }, { "clip_ratio": 0.0, "completion_length": 663.7500228881836, "epoch": 0.024666666666666667, "grad_norm": 0.16912373900413513, "kl": 0.00013953447341918945, "learning_rate": 7.4e-07, "loss": -0.0184, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 37 }, { "clip_ratio": 0.0, "completion_length": 646.7916793823242, "epoch": 0.025333333333333333, "grad_norm": 0.16238808631896973, "kl": 0.00023627281188964844, "learning_rate": 7.600000000000001e-07, "loss": 0.0679, "reward": 0.0833333358168602, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 38 }, { "clip_ratio": 0.0, "completion_length": 548.8333358764648, "epoch": 0.026, "grad_norm": 0.2710413932800293, "kl": 0.00025725364685058594, "learning_rate": 7.8e-07, "loss": -0.0228, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 39 }, { "clip_ratio": 0.0, "completion_length": 500.8750228881836, "epoch": 0.02666666666666667, "grad_norm": 0.0019501454662531614, "kl": 0.00027489662170410156, "learning_rate": 8e-07, "loss": 0.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 40 }, { "clip_ratio": 0.0, "completion_length": 457.3333511352539, "epoch": 0.027333333333333334, "grad_norm": 0.3987518846988678, "kl": 0.0004105567932128906, "learning_rate": 8.2e-07, "loss": -0.1253, "reward": 0.1250000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 41 }, { "clip_ratio": 0.0, "completion_length": 459.1666793823242, "epoch": 0.028, "grad_norm": 0.002402559155598283, "kl": 0.00024962425231933594, "learning_rate": 8.400000000000001e-07, "loss": 0.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 42 }, { "clip_ratio": 0.0, "completion_length": 649.9583435058594, "epoch": 0.028666666666666667, "grad_norm": 0.25429248809814453, "kl": 0.00029921531677246094, "learning_rate": 8.6e-07, "loss": 0.1439, "reward": 0.0833333358168602, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0416666679084301, "step": 43 }, { "clip_ratio": 0.0, "completion_length": 484.0833511352539, "epoch": 0.029333333333333333, "grad_norm": 0.27706989645957947, "kl": 0.0005125999450683594, "learning_rate": 8.8e-07, "loss": -0.0183, "reward": 0.0416666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 44 }, { "clip_ratio": 0.0, "completion_length": 531.6250152587891, "epoch": 0.03, "grad_norm": 0.4644177556037903, "kl": 0.00035762786865234375, "learning_rate": 9e-07, "loss": -0.0525, "reward": 0.1666666716337204, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 45 }, { "clip_ratio": 0.0, "completion_length": 556.2916870117188, "epoch": 0.030666666666666665, "grad_norm": 0.0032004474196583033, "kl": 0.0004756450653076172, "learning_rate": 9.2e-07, "loss": 0.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 46 }, { "clip_ratio": 0.0, "completion_length": 689.7500152587891, "epoch": 0.03133333333333333, "grad_norm": 0.42621853947639465, "kl": 0.0002739429473876953, "learning_rate": 9.400000000000001e-07, "loss": 0.0182, "reward": 0.0416666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 47 }, { "clip_ratio": 0.0, "completion_length": 419.2083435058594, "epoch": 0.032, "grad_norm": 0.12698478996753693, "kl": 0.0006108283996582031, "learning_rate": 9.600000000000001e-07, "loss": 0.0383, "reward": 0.0416666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 48 }, { "clip_ratio": 0.0, "completion_length": 504.62501525878906, "epoch": 0.03266666666666666, "grad_norm": 0.23512697219848633, "kl": 0.0005581378936767578, "learning_rate": 9.8e-07, "loss": 0.0795, "reward": 0.0833333358168602, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 49 }, { "clip_ratio": 0.0, "completion_length": 531.0000152587891, "epoch": 0.03333333333333333, "grad_norm": 0.1959124356508255, "kl": 0.0008521080017089844, "learning_rate": 1e-06, "loss": 0.0173, "reward": 0.0833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 50 }, { "clip_ratio": 0.0, "completion_length": 454.0000228881836, "epoch": 0.034, "grad_norm": 0.24467039108276367, "kl": 0.0006885528564453125, "learning_rate": 1.0200000000000002e-06, "loss": -0.0794, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 51 }, { "clip_ratio": 0.0, "completion_length": 386.7083435058594, "epoch": 0.034666666666666665, "grad_norm": 0.20513416826725006, "kl": 0.0005373954772949219, "learning_rate": 1.04e-06, "loss": 0.0, "reward": 0.0833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 52 }, { "clip_ratio": 0.0, "completion_length": 500.625, "epoch": 0.035333333333333335, "grad_norm": 0.2584396004676819, "kl": 0.0005807876586914062, "learning_rate": 1.06e-06, "loss": -0.1054, "reward": 0.0833333358168602, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 53 }, { "clip_ratio": 0.0, "completion_length": 549.3333435058594, "epoch": 0.036, "grad_norm": 0.2827392518520355, "kl": 0.0003368854522705078, "learning_rate": 1.08e-06, "loss": -0.0098, "reward": 0.125, "reward_std": 0.1369306445121765, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0416666679084301, "step": 54 }, { "clip_ratio": 0.0, "completion_length": 550.9583435058594, "epoch": 0.03666666666666667, "grad_norm": 0.20813292264938354, "kl": 0.00035691261291503906, "learning_rate": 1.0999999999999998e-06, "loss": 0.058, "reward": 0.0833333358168602, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 55 }, { "clip_ratio": 0.0, "completion_length": 459.3333435058594, "epoch": 0.037333333333333336, "grad_norm": 0.2745841145515442, "kl": 0.0008111000061035156, "learning_rate": 1.12e-06, "loss": -0.1037, "reward": 0.0833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 56 }, { "clip_ratio": 0.0, "completion_length": 534.9166717529297, "epoch": 0.038, "grad_norm": 0.004238252993673086, "kl": 0.0015835762023925781, "learning_rate": 1.14e-06, "loss": 0.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 57 }, { "clip_ratio": 0.0, "completion_length": 641.7083587646484, "epoch": 0.03866666666666667, "grad_norm": 0.34064215421676636, "kl": 0.0005714893341064453, "learning_rate": 1.16e-06, "loss": -0.1213, "reward": 0.1666666716337204, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 58 }, { "clip_ratio": 0.0, "completion_length": 552.3750076293945, "epoch": 0.03933333333333333, "grad_norm": 0.34779950976371765, "kl": 0.0011487007141113281, "learning_rate": 1.18e-06, "loss": 0.0644, "reward": 0.1666666716337204, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 59 }, { "clip_ratio": 0.0, "completion_length": 707.2916946411133, "epoch": 0.04, "grad_norm": 0.5549023151397705, "kl": 0.0006215572357177734, "learning_rate": 1.2000000000000002e-06, "loss": -0.0164, "reward": 0.1250000037252903, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0416666679084301, "step": 60 }, { "clip_ratio": 0.0, "completion_length": 539.0416793823242, "epoch": 0.04066666666666666, "grad_norm": 0.26132234930992126, "kl": 0.0009131431579589844, "learning_rate": 1.22e-06, "loss": 0.09, "reward": 0.1250000037252903, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 61 }, { "clip_ratio": 0.0, "completion_length": 486.08333587646484, "epoch": 0.04133333333333333, "grad_norm": 0.2925982177257538, "kl": 0.0009722709655761719, "learning_rate": 1.24e-06, "loss": 0.0489, "reward": 0.1250000037252903, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 62 }, { "clip_ratio": 0.0, "completion_length": 534.1666793823242, "epoch": 0.042, "grad_norm": 0.17420487105846405, "kl": 0.0035161972045898438, "learning_rate": 1.26e-06, "loss": 0.0738, "reward": 0.0416666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 63 }, { "clip_ratio": 0.0, "completion_length": 599.5833358764648, "epoch": 0.042666666666666665, "grad_norm": 0.2061598002910614, "kl": 0.0008149147033691406, "learning_rate": 1.28e-06, "loss": -0.0428, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 64 }, { "clip_ratio": 0.0, "completion_length": 622.2916717529297, "epoch": 0.043333333333333335, "grad_norm": 0.18498694896697998, "kl": 0.00124359130859375, "learning_rate": 1.3e-06, "loss": 0.0367, "reward": 0.0416666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 65 }, { "clip_ratio": 0.0, "completion_length": 465.9166793823242, "epoch": 0.044, "grad_norm": 0.26345258951187134, "kl": 0.0015192031860351562, "learning_rate": 1.32e-06, "loss": -0.0575, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 66 }, { "clip_ratio": 0.0, "completion_length": 586.3750305175781, "epoch": 0.04466666666666667, "grad_norm": 0.00507685961201787, "kl": 0.0015087127685546875, "learning_rate": 1.34e-06, "loss": 0.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 67 }, { "clip_ratio": 0.0, "completion_length": 488.5000190734863, "epoch": 0.04533333333333334, "grad_norm": 0.01482386328279972, "kl": 0.0021941661834716797, "learning_rate": 1.36e-06, "loss": 0.0001, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 68 }, { "clip_ratio": 0.0, "completion_length": 706.9583435058594, "epoch": 0.046, "grad_norm": 0.16949452459812164, "kl": 0.0009508132934570312, "learning_rate": 1.3800000000000001e-06, "loss": 0.073, "reward": 0.2083333395421505, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 69 }, { "clip_ratio": 0.0, "completion_length": 533.2500228881836, "epoch": 0.04666666666666667, "grad_norm": 0.1964757740497589, "kl": 0.00174713134765625, "learning_rate": 1.4000000000000001e-06, "loss": -0.021, "reward": 0.0833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 70 }, { "clip_ratio": 0.0, "completion_length": 510.12501525878906, "epoch": 0.04733333333333333, "grad_norm": 0.19148533046245575, "kl": 0.002109527587890625, "learning_rate": 1.42e-06, "loss": 0.1422, "reward": 0.0833333358168602, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0416666679084301, "step": 71 }, { "clip_ratio": 0.0, "completion_length": 562.5416870117188, "epoch": 0.048, "grad_norm": 0.22625623643398285, "kl": 0.0015139579772949219, "learning_rate": 1.44e-06, "loss": -0.0632, "reward": 0.0416666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 72 }, { "clip_ratio": 0.0, "completion_length": 691.2916717529297, "epoch": 0.048666666666666664, "grad_norm": 0.004117187112569809, "kl": 0.0011053085327148438, "learning_rate": 1.4600000000000002e-06, "loss": 0.0, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 73 }, { "clip_ratio": 0.0, "completion_length": 509.50002670288086, "epoch": 0.04933333333333333, "grad_norm": 0.19351254403591156, "kl": 0.00386810302734375, "learning_rate": 1.48e-06, "loss": 0.0183, "reward": 0.0416666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 74 }, { "clip_ratio": 0.0, "completion_length": 469.0833511352539, "epoch": 0.05, "grad_norm": 0.37108704447746277, "kl": 0.0026836395263671875, "learning_rate": 1.5e-06, "loss": -0.0166, "reward": 0.0833333358168602, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 75 }, { "clip_ratio": 0.0, "completion_length": 257.0833435058594, "epoch": 0.050666666666666665, "grad_norm": 0.45365118980407715, "kl": 0.00313568115234375, "learning_rate": 1.5200000000000003e-06, "loss": -0.0997, "reward": 0.1666666679084301, "reward_std": 0.23899272084236145, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 76 }, { "clip_ratio": 0.0, "completion_length": 657.0416870117188, "epoch": 0.051333333333333335, "grad_norm": 0.22891730070114136, "kl": 0.0016660690307617188, "learning_rate": 1.5399999999999999e-06, "loss": -0.0748, "reward": 0.0833333358168602, "reward_std": 0.20412415266036987, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 77 }, { "clip_ratio": 0.0, "completion_length": 630.7916793823242, "epoch": 0.052, "grad_norm": 0.09813319891691208, "kl": 0.0015506744384765625, "learning_rate": 1.56e-06, "loss": 0.0547, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 78 }, { "clip_ratio": 0.0, "completion_length": 454.8333435058594, "epoch": 0.05266666666666667, "grad_norm": 0.12848494946956635, "kl": 0.00341796875, "learning_rate": 1.58e-06, "loss": 0.0382, "reward": 0.0416666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 79 }, { "clip_ratio": 0.0, "completion_length": 516.0416870117188, "epoch": 0.05333333333333334, "grad_norm": 0.012212786823511124, "kl": 0.00270843505859375, "learning_rate": 1.6e-06, "loss": 0.0001, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 80 }, { "clip_ratio": 0.0, "completion_length": 597.7916870117188, "epoch": 0.054, "grad_norm": 0.006705094128847122, "kl": 0.0016918182373046875, "learning_rate": 1.6200000000000002e-06, "loss": 0.0001, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 81 }, { "clip_ratio": 0.0, "completion_length": 547.5416870117188, "epoch": 0.05466666666666667, "grad_norm": 0.3336743116378784, "kl": 0.003910064697265625, "learning_rate": 1.64e-06, "loss": 0.0212, "reward": 0.2500000074505806, "reward_std": 0.46232305467128754, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 82 }, { "clip_ratio": 0.0, "completion_length": 658.3333511352539, "epoch": 0.05533333333333333, "grad_norm": 0.20677541196346283, "kl": 0.0025272369384765625, "learning_rate": 1.66e-06, "loss": -0.0226, "reward": 0.0833333358168602, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 83 }, { "clip_ratio": 0.0, "completion_length": 509.5000228881836, "epoch": 0.056, "grad_norm": 0.17898258566856384, "kl": 0.00299072265625, "learning_rate": 1.6800000000000002e-06, "loss": 0.0017, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 84 }, { "clip_ratio": 0.0, "completion_length": 469.7083511352539, "epoch": 0.056666666666666664, "grad_norm": 2.457906723022461, "kl": 0.01364898681640625, "learning_rate": 1.7e-06, "loss": -0.0917, "reward": 0.2916666716337204, "reward_std": 0.4701542481780052, "rewards/accuracy_reward": 0.2916666716337204, "rewards/format_reward": 0.0, "step": 85 }, { "clip_ratio": 0.0, "completion_length": 662.7916793823242, "epoch": 0.05733333333333333, "grad_norm": 0.1375531405210495, "kl": 0.00433349609375, "learning_rate": 1.72e-06, "loss": 0.0289, "reward": 0.0833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 86 }, { "clip_ratio": 0.0, "completion_length": 657.5, "epoch": 0.058, "grad_norm": 0.007807570043951273, "kl": 0.00315093994140625, "learning_rate": 1.7399999999999999e-06, "loss": 0.0001, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 87 }, { "clip_ratio": 0.0, "completion_length": 658.7083587646484, "epoch": 0.058666666666666666, "grad_norm": 0.1905287355184555, "kl": 0.002803802490234375, "learning_rate": 1.76e-06, "loss": 0.0212, "reward": 0.125, "reward_std": 0.1369306445121765, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "step": 88 }, { "clip_ratio": 0.0, "completion_length": 668.7916870117188, "epoch": 0.059333333333333335, "grad_norm": 0.01212354563176632, "kl": 0.005977630615234375, "learning_rate": 1.7800000000000001e-06, "loss": 0.0001, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 89 }, { "clip_ratio": 0.0, "completion_length": 644.1250228881836, "epoch": 0.06, "grad_norm": 0.09124124050140381, "kl": 0.01165008544921875, "learning_rate": 1.8e-06, "loss": 0.0003, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 90 }, { "clip_ratio": 0.0, "completion_length": 497.37500762939453, "epoch": 0.06066666666666667, "grad_norm": 0.3951278030872345, "kl": 0.005802154541015625, "learning_rate": 1.8200000000000002e-06, "loss": 0.0588, "reward": 0.1250000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 91 }, { "clip_ratio": 0.0, "completion_length": 548.3750228881836, "epoch": 0.06133333333333333, "grad_norm": 0.32031112909317017, "kl": 0.009769439697265625, "learning_rate": 1.84e-06, "loss": 0.0763, "reward": 0.1250000037252903, "reward_std": 0.3061862215399742, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 92 }, { "clip_ratio": 0.0, "completion_length": 685.1250305175781, "epoch": 0.062, "grad_norm": 0.006467781029641628, "kl": 0.0028171539306640625, "learning_rate": 1.86e-06, "loss": 0.0001, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 93 }, { "clip_ratio": 0.0, "completion_length": 843.6666870117188, "epoch": 0.06266666666666666, "grad_norm": 0.09722902625799179, "kl": 0.00466156005859375, "learning_rate": 1.8800000000000002e-06, "loss": 0.0892, "reward": 0.0416666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 94 }, { "clip_ratio": 0.0, "completion_length": 372.1250114440918, "epoch": 0.06333333333333334, "grad_norm": 0.29740825295448303, "kl": 0.010781288146972656, "learning_rate": 1.9e-06, "loss": 0.0194, "reward": 0.0833333358168602, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 95 }, { "clip_ratio": 0.0, "completion_length": 661.6250305175781, "epoch": 0.064, "grad_norm": 0.25428321957588196, "kl": 0.004550933837890625, "learning_rate": 1.9200000000000003e-06, "loss": -0.0254, "reward": 0.125, "reward_std": 0.1369306445121765, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "step": 96 }, { "clip_ratio": 0.0, "completion_length": 525.3333511352539, "epoch": 0.06466666666666666, "grad_norm": 0.32635006308555603, "kl": 0.02454376220703125, "learning_rate": 1.94e-06, "loss": -0.0188, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 97 }, { "clip_ratio": 0.0, "completion_length": 677.1250152587891, "epoch": 0.06533333333333333, "grad_norm": 0.17032942175865173, "kl": 0.0036869049072265625, "learning_rate": 1.96e-06, "loss": 0.0291, "reward": 0.0833333358168602, "reward_std": 0.20412415266036987, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 98 }, { "clip_ratio": 0.0, "completion_length": 737.0000152587891, "epoch": 0.066, "grad_norm": 0.012928685173392296, "kl": 0.01392364501953125, "learning_rate": 1.98e-06, "loss": 0.0001, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 99 }, { "clip_ratio": 0.0, "completion_length": 636.8750228881836, "epoch": 0.06666666666666667, "grad_norm": 0.21052291989326477, "kl": 0.0057220458984375, "learning_rate": 2e-06, "loss": 0.0339, "reward": 0.1250000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 100 }, { "clip_ratio": 0.0, "completion_length": 730.2500152587891, "epoch": 0.06733333333333333, "grad_norm": 0.23326906561851501, "kl": 0.008758544921875, "learning_rate": 2.02e-06, "loss": 0.0012, "reward": 0.0416666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 101 }, { "clip_ratio": 0.0, "completion_length": 604.6250152587891, "epoch": 0.068, "grad_norm": 0.15730462968349457, "kl": 0.00848388671875, "learning_rate": 2.0400000000000004e-06, "loss": 0.0336, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 102 }, { "clip_ratio": 0.0, "completion_length": 853.7083435058594, "epoch": 0.06866666666666667, "grad_norm": 0.18045632541179657, "kl": 0.0026493072509765625, "learning_rate": 2.06e-06, "loss": 0.0022, "reward": 0.0833333358168602, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 103 }, { "clip_ratio": 0.0, "completion_length": 717.7916870117188, "epoch": 0.06933333333333333, "grad_norm": 0.18273942172527313, "kl": 0.00449371337890625, "learning_rate": 2.08e-06, "loss": 0.0721, "reward": 0.1250000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 104 }, { "clip_ratio": 0.0, "completion_length": 653.6250152587891, "epoch": 0.07, "grad_norm": 0.14138540625572205, "kl": 0.00603485107421875, "learning_rate": 2.1e-06, "loss": 0.0364, "reward": 0.0416666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 105 }, { "clip_ratio": 0.0, "completion_length": 575.3333435058594, "epoch": 0.07066666666666667, "grad_norm": 0.10860440880060196, "kl": 0.00777435302734375, "learning_rate": 2.12e-06, "loss": 0.0655, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 106 }, { "clip_ratio": 0.0, "completion_length": 841.1666870117188, "epoch": 0.07133333333333333, "grad_norm": 0.07324881851673126, "kl": 0.007228851318359375, "learning_rate": 2.1400000000000003e-06, "loss": 0.022, "reward": 0.0416666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 107 }, { "clip_ratio": 0.0, "completion_length": 609.3333435058594, "epoch": 0.072, "grad_norm": 0.15986397862434387, "kl": 0.00608062744140625, "learning_rate": 2.16e-06, "loss": 0.0267, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 108 }, { "clip_ratio": 0.0, "completion_length": 679.2500152587891, "epoch": 0.07266666666666667, "grad_norm": 0.022387860342860222, "kl": 0.0064697265625, "learning_rate": 2.1800000000000003e-06, "loss": 0.0002, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 109 }, { "clip_ratio": 0.0, "completion_length": 504.1666793823242, "epoch": 0.07333333333333333, "grad_norm": 0.32860374450683594, "kl": 0.006134033203125, "learning_rate": 2.1999999999999997e-06, "loss": -0.1785, "reward": 0.1666666679084301, "reward_std": 0.23899272084236145, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 110 }, { "clip_ratio": 0.0, "completion_length": 673.0833587646484, "epoch": 0.074, "grad_norm": 0.12361077219247818, "kl": 0.004840850830078125, "learning_rate": 2.22e-06, "loss": 0.0341, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 111 }, { "clip_ratio": 0.0, "completion_length": 593.8750152587891, "epoch": 0.07466666666666667, "grad_norm": 0.14434115588665009, "kl": 0.007175445556640625, "learning_rate": 2.24e-06, "loss": 0.0687, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 112 }, { "clip_ratio": 0.0, "completion_length": 635.3333587646484, "epoch": 0.07533333333333334, "grad_norm": 0.18591949343681335, "kl": 0.01331329345703125, "learning_rate": 2.26e-06, "loss": 0.0706, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 113 }, { "clip_ratio": 0.0, "completion_length": 619.7500152587891, "epoch": 0.076, "grad_norm": 0.11987698078155518, "kl": 0.00876617431640625, "learning_rate": 2.28e-06, "loss": 0.0634, "reward": 0.0416666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 114 }, { "clip_ratio": 0.0, "completion_length": 810.0416870117188, "epoch": 0.07666666666666666, "grad_norm": 0.183257058262825, "kl": 0.005126953125, "learning_rate": 2.3000000000000004e-06, "loss": -0.0497, "reward": 0.1250000037252903, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 115 }, { "clip_ratio": 0.0, "completion_length": 724.8333511352539, "epoch": 0.07733333333333334, "grad_norm": 0.14316679537296295, "kl": 0.00836181640625, "learning_rate": 2.32e-06, "loss": 0.0759, "reward": 0.0833333358168602, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 116 }, { "clip_ratio": 0.0, "completion_length": 784.0833587646484, "epoch": 0.078, "grad_norm": 0.15028031170368195, "kl": 0.006526947021484375, "learning_rate": 2.34e-06, "loss": 0.0874, "reward": 0.0833333358168602, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 117 }, { "clip_ratio": 0.0, "completion_length": 557.5416717529297, "epoch": 0.07866666666666666, "grad_norm": 0.2742733061313629, "kl": 0.01168060302734375, "learning_rate": 2.36e-06, "loss": -0.0395, "reward": 0.0833333358168602, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 118 }, { "clip_ratio": 0.0, "completion_length": 786.2083587646484, "epoch": 0.07933333333333334, "grad_norm": 0.015722224488854408, "kl": 0.0079803466796875, "learning_rate": 2.38e-06, "loss": 0.0002, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 119 }, { "clip_ratio": 0.0, "completion_length": 643.3750152587891, "epoch": 0.08, "grad_norm": 0.2190222293138504, "kl": 0.01145172119140625, "learning_rate": 2.4000000000000003e-06, "loss": -0.0005, "reward": 0.1666666716337204, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 120 }, { "clip_ratio": 0.0, "completion_length": 638.0833587646484, "epoch": 0.08066666666666666, "grad_norm": 0.26958101987838745, "kl": 0.0125885009765625, "learning_rate": 2.42e-06, "loss": 0.0029, "reward": 0.1250000037252903, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 121 }, { "clip_ratio": 0.0, "completion_length": 722.6666870117188, "epoch": 0.08133333333333333, "grad_norm": 0.09279055893421173, "kl": 0.02370452880859375, "learning_rate": 2.44e-06, "loss": 0.0595, "reward": 0.0833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 122 }, { "clip_ratio": 0.0, "completion_length": 594.8333435058594, "epoch": 0.082, "grad_norm": 0.30548200011253357, "kl": 0.0137939453125, "learning_rate": 2.4599999999999997e-06, "loss": -0.0188, "reward": 0.2500000074505806, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 123 }, { "clip_ratio": 0.0, "completion_length": 610.6666793823242, "epoch": 0.08266666666666667, "grad_norm": 0.22042034566402435, "kl": 0.0106353759765625, "learning_rate": 2.48e-06, "loss": -0.0412, "reward": 0.291666679084301, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.291666679084301, "rewards/format_reward": 0.0, "step": 124 }, { "clip_ratio": 0.0, "completion_length": 663.3333511352539, "epoch": 0.08333333333333333, "grad_norm": 0.1725771576166153, "kl": 0.1601104736328125, "learning_rate": 2.5e-06, "loss": 0.014, "reward": 0.1666666716337204, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 125 }, { "clip_ratio": 0.0, "completion_length": 707.2916870117188, "epoch": 0.084, "grad_norm": 0.20420992374420166, "kl": 0.0082550048828125, "learning_rate": 2.52e-06, "loss": 0.011, "reward": 0.1666666679084301, "reward_std": 0.23899272084236145, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 126 }, { "clip_ratio": 0.0, "completion_length": 658.7916870117188, "epoch": 0.08466666666666667, "grad_norm": 0.17936432361602783, "kl": 0.0102386474609375, "learning_rate": 2.5400000000000002e-06, "loss": -0.0081, "reward": 0.0833333358168602, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 127 }, { "clip_ratio": 0.0, "completion_length": 689.7083435058594, "epoch": 0.08533333333333333, "grad_norm": 0.1169237345457077, "kl": 0.0149993896484375, "learning_rate": 2.56e-06, "loss": 0.0619, "reward": 0.125, "reward_std": 0.1369306445121765, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "step": 128 }, { "clip_ratio": 0.0, "completion_length": 645.8333511352539, "epoch": 0.086, "grad_norm": 0.29634037613868713, "kl": 0.01059722900390625, "learning_rate": 2.58e-06, "loss": 0.0547, "reward": 0.2916666753590107, "reward_std": 0.43528567999601364, "rewards/accuracy_reward": 0.2916666753590107, "rewards/format_reward": 0.0, "step": 129 }, { "clip_ratio": 0.0, "completion_length": 724.5416870117188, "epoch": 0.08666666666666667, "grad_norm": 0.17030656337738037, "kl": 0.013671875, "learning_rate": 2.6e-06, "loss": 0.0154, "reward": 0.0833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 130 }, { "clip_ratio": 0.0, "completion_length": 685.9583435058594, "epoch": 0.08733333333333333, "grad_norm": 0.188010111451149, "kl": 0.01800537109375, "learning_rate": 2.62e-06, "loss": 0.0602, "reward": 0.1666666716337204, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 131 }, { "clip_ratio": 0.0, "completion_length": 687.3333587646484, "epoch": 0.088, "grad_norm": 0.28396326303482056, "kl": 0.03287506103515625, "learning_rate": 2.64e-06, "loss": -0.0038, "reward": 0.3333333358168602, "reward_std": 0.4779854267835617, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 132 }, { "clip_ratio": 0.0, "completion_length": 688.6250305175781, "epoch": 0.08866666666666667, "grad_norm": 0.07562713325023651, "kl": 0.016754150390625, "learning_rate": 2.6600000000000004e-06, "loss": 0.006, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 133 }, { "clip_ratio": 0.0, "completion_length": 726.1250152587891, "epoch": 0.08933333333333333, "grad_norm": 0.1273273080587387, "kl": 0.0218505859375, "learning_rate": 2.68e-06, "loss": 0.0002, "reward": 0.0833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 134 }, { "clip_ratio": 0.0, "completion_length": 708.9583435058594, "epoch": 0.09, "grad_norm": 0.09302831441164017, "kl": 0.0112762451171875, "learning_rate": 2.7e-06, "loss": 0.022, "reward": 0.0416666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 135 }, { "clip_ratio": 0.0, "completion_length": 576.7916870117188, "epoch": 0.09066666666666667, "grad_norm": 0.10915770381689072, "kl": 0.050048828125, "learning_rate": 2.72e-06, "loss": 0.0341, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 136 }, { "clip_ratio": 0.0, "completion_length": 679.3333587646484, "epoch": 0.09133333333333334, "grad_norm": 0.2542513310909271, "kl": 0.01689910888671875, "learning_rate": 2.74e-06, "loss": -0.0052, "reward": 0.2083333395421505, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 137 }, { "clip_ratio": 0.0, "completion_length": 521.9583549499512, "epoch": 0.092, "grad_norm": 0.14885935187339783, "kl": 0.024444580078125, "learning_rate": 2.7600000000000003e-06, "loss": 0.1036, "reward": 0.3333333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 138 }, { "clip_ratio": 0.0, "completion_length": 687.2083587646484, "epoch": 0.09266666666666666, "grad_norm": 0.21206925809383392, "kl": 0.0339202880859375, "learning_rate": 2.78e-06, "loss": 0.1327, "reward": 0.1666666716337204, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 139 }, { "clip_ratio": 0.0, "completion_length": 690.8333587646484, "epoch": 0.09333333333333334, "grad_norm": 0.2633911967277527, "kl": 0.019989013671875, "learning_rate": 2.8000000000000003e-06, "loss": 0.0505, "reward": 0.2500000037252903, "reward_std": 0.3680921792984009, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.0, "step": 140 }, { "clip_ratio": 0.0, "completion_length": 621.4583435058594, "epoch": 0.094, "grad_norm": 0.19379253685474396, "kl": 0.0284271240234375, "learning_rate": 2.82e-06, "loss": 0.0496, "reward": 0.2500000111758709, "reward_std": 0.20412415266036987, "rewards/accuracy_reward": 0.2500000111758709, "rewards/format_reward": 0.0, "step": 141 }, { "clip_ratio": 0.0, "completion_length": 592.7500076293945, "epoch": 0.09466666666666666, "grad_norm": 0.24894554913043976, "kl": 0.0223846435546875, "learning_rate": 2.84e-06, "loss": 0.1377, "reward": 0.2083333395421505, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 142 }, { "clip_ratio": 0.0, "completion_length": 619.2083358764648, "epoch": 0.09533333333333334, "grad_norm": 0.5328335165977478, "kl": 0.0306854248046875, "learning_rate": 2.86e-06, "loss": -0.0207, "reward": 0.3750000037252903, "reward_std": 0.5050228238105774, "rewards/accuracy_reward": 0.3750000037252903, "rewards/format_reward": 0.0, "step": 143 }, { "clip_ratio": 0.0, "completion_length": 721.6666870117188, "epoch": 0.096, "grad_norm": 0.2416180819272995, "kl": 0.016448974609375, "learning_rate": 2.88e-06, "loss": 0.0367, "reward": 0.2500000037252903, "reward_std": 0.44311685860157013, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.0, "step": 144 }, { "clip_ratio": 0.0, "completion_length": 653.2500152587891, "epoch": 0.09666666666666666, "grad_norm": 0.1984442174434662, "kl": 0.04180908203125, "learning_rate": 2.9e-06, "loss": 0.0475, "reward": 0.2083333395421505, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 145 }, { "clip_ratio": 0.0, "completion_length": 599.3333358764648, "epoch": 0.09733333333333333, "grad_norm": 0.1903398185968399, "kl": 0.031341552734375, "learning_rate": 2.9200000000000004e-06, "loss": -0.0522, "reward": 0.2083333358168602, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.2083333358168602, "rewards/format_reward": 0.0, "step": 146 }, { "clip_ratio": 0.0, "completion_length": 747.5000305175781, "epoch": 0.098, "grad_norm": 0.19534535706043243, "kl": 0.01983642578125, "learning_rate": 2.9400000000000002e-06, "loss": 0.012, "reward": 0.0833333358168602, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 147 }, { "clip_ratio": 0.0, "completion_length": 568.5833435058594, "epoch": 0.09866666666666667, "grad_norm": 0.4018630385398865, "kl": 0.0225677490234375, "learning_rate": 2.96e-06, "loss": 0.1331, "reward": 0.5000000149011612, "reward_std": 0.5163978338241577, "rewards/accuracy_reward": 0.5000000149011612, "rewards/format_reward": 0.0, "step": 148 }, { "clip_ratio": 0.0, "completion_length": 494.8750305175781, "epoch": 0.09933333333333333, "grad_norm": 0.2582593858242035, "kl": 0.023284912109375, "learning_rate": 2.98e-06, "loss": 0.1343, "reward": 0.3750000111758709, "reward_std": 0.3410547822713852, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 149 }, { "clip_ratio": 0.0, "completion_length": 582.6666870117188, "epoch": 0.1, "grad_norm": 0.210843026638031, "kl": 0.03057861328125, "learning_rate": 3e-06, "loss": 0.0329, "reward": 0.1250000037252903, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 150 }, { "clip_ratio": 0.0, "completion_length": 630.4166717529297, "epoch": 0.10066666666666667, "grad_norm": 0.3218689262866974, "kl": 0.0461273193359375, "learning_rate": 2.999995938436236e-06, "loss": 0.0259, "reward": 0.2083333395421505, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 151 }, { "clip_ratio": 0.0, "completion_length": 543.6666793823242, "epoch": 0.10133333333333333, "grad_norm": 0.3358828127384186, "kl": 0.02095794677734375, "learning_rate": 2.9999837537669383e-06, "loss": 0.0668, "reward": 0.6250000223517418, "reward_std": 0.4893604442477226, "rewards/accuracy_reward": 0.6250000223517418, "rewards/format_reward": 0.0, "step": 152 }, { "clip_ratio": 0.0, "completion_length": 588.0000152587891, "epoch": 0.102, "grad_norm": 0.3361766040325165, "kl": 0.03912353515625, "learning_rate": 2.999963446058092e-06, "loss": 0.0732, "reward": 0.3333333432674408, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 153 }, { "clip_ratio": 0.0, "completion_length": 559.4166793823242, "epoch": 0.10266666666666667, "grad_norm": 0.14912772178649902, "kl": 0.01708984375, "learning_rate": 2.9999350154196726e-06, "loss": -0.0043, "reward": 0.0416666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 154 }, { "clip_ratio": 0.0, "completion_length": 630.5833435058594, "epoch": 0.10333333333333333, "grad_norm": 0.19982485473155975, "kl": 0.018524169921875, "learning_rate": 2.9998984620056434e-06, "loss": 0.0499, "reward": 0.3750000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.3750000037252903, "rewards/format_reward": 0.0, "step": 155 }, { "clip_ratio": 0.0, "completion_length": 640.8750152587891, "epoch": 0.104, "grad_norm": 0.17891466617584229, "kl": 0.0235137939453125, "learning_rate": 2.9998537860139563e-06, "loss": 0.0786, "reward": 0.0833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 156 }, { "clip_ratio": 0.0, "completion_length": 527.9166717529297, "epoch": 0.10466666666666667, "grad_norm": 0.2621549367904663, "kl": 0.0246124267578125, "learning_rate": 2.9998009876865506e-06, "loss": 0.0958, "reward": 0.416666679084301, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.416666679084301, "rewards/format_reward": 0.0, "step": 157 }, { "clip_ratio": 0.0, "completion_length": 649.5416793823242, "epoch": 0.10533333333333333, "grad_norm": 0.28704845905303955, "kl": 0.0178375244140625, "learning_rate": 2.9997400673093517e-06, "loss": 0.023, "reward": 0.2916666716337204, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.2916666716337204, "rewards/format_reward": 0.0, "step": 158 }, { "clip_ratio": 0.0, "completion_length": 621.708366394043, "epoch": 0.106, "grad_norm": 0.22550106048583984, "kl": 0.015716552734375, "learning_rate": 2.9996710252122682e-06, "loss": -0.0013, "reward": 0.2083333358168602, "reward_std": 0.3410547822713852, "rewards/accuracy_reward": 0.2083333358168602, "rewards/format_reward": 0.0, "step": 159 }, { "clip_ratio": 0.0, "completion_length": 577.6666793823242, "epoch": 0.10666666666666667, "grad_norm": 0.25608718395233154, "kl": 0.0174102783203125, "learning_rate": 2.9995938617691924e-06, "loss": 0.0426, "reward": 0.2916666716337204, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.2916666716337204, "rewards/format_reward": 0.0, "step": 160 }, { "clip_ratio": 0.0, "completion_length": 723.5000152587891, "epoch": 0.10733333333333334, "grad_norm": 0.2432831972837448, "kl": 0.019622802734375, "learning_rate": 2.9995085773979966e-06, "loss": 0.0463, "reward": 0.2083333395421505, "reward_std": 0.43528568744659424, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 161 }, { "clip_ratio": 0.0, "completion_length": 473.4583511352539, "epoch": 0.108, "grad_norm": 0.2569054067134857, "kl": 0.04974365234375, "learning_rate": 2.9994151725605313e-06, "loss": 0.0532, "reward": 0.3750000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.3750000037252903, "rewards/format_reward": 0.0, "step": 162 }, { "clip_ratio": 0.0, "completion_length": 577.3750152587891, "epoch": 0.10866666666666666, "grad_norm": 0.21905206143856049, "kl": 0.02850341796875, "learning_rate": 2.999313647762623e-06, "loss": -0.0084, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 163 }, { "clip_ratio": 0.0, "completion_length": 629.5416946411133, "epoch": 0.10933333333333334, "grad_norm": 0.17997115850448608, "kl": 0.026947021484375, "learning_rate": 2.9992040035540708e-06, "loss": 0.0483, "reward": 0.2500000074505806, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 164 }, { "clip_ratio": 0.0, "completion_length": 610.2500152587891, "epoch": 0.11, "grad_norm": 0.32188573479652405, "kl": 0.0284271240234375, "learning_rate": 2.9990862405286437e-06, "loss": 0.0362, "reward": 0.2500000037252903, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.0, "step": 165 }, { "clip_ratio": 0.0, "completion_length": 471.50000762939453, "epoch": 0.11066666666666666, "grad_norm": 0.38114094734191895, "kl": 0.05401611328125, "learning_rate": 2.9989603593240777e-06, "loss": -0.0331, "reward": 0.1250000037252903, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 166 }, { "clip_ratio": 0.0, "completion_length": 553.7500152587891, "epoch": 0.11133333333333334, "grad_norm": 0.1695621907711029, "kl": 0.030792236328125, "learning_rate": 2.9988263606220727e-06, "loss": 0.0082, "reward": 0.2083333432674408, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.2083333432674408, "rewards/format_reward": 0.0, "step": 167 }, { "clip_ratio": 0.0, "completion_length": 634.4583587646484, "epoch": 0.112, "grad_norm": 0.09795773029327393, "kl": 0.02398681640625, "learning_rate": 2.9986842451482876e-06, "loss": -0.0399, "reward": 0.125, "reward_std": 0.1369306445121765, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "step": 168 }, { "clip_ratio": 0.0, "completion_length": 482.6666717529297, "epoch": 0.11266666666666666, "grad_norm": 0.24294187128543854, "kl": 0.019989013671875, "learning_rate": 2.998534013672337e-06, "loss": 0.0512, "reward": 0.4583333432674408, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.4583333432674408, "rewards/format_reward": 0.0, "step": 169 }, { "clip_ratio": 0.0, "completion_length": 452.75001525878906, "epoch": 0.11333333333333333, "grad_norm": 0.35981622338294983, "kl": 0.039886474609375, "learning_rate": 2.998375667007787e-06, "loss": -0.0565, "reward": 0.1666666716337204, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 170 }, { "clip_ratio": 0.0, "completion_length": 486.95835876464844, "epoch": 0.114, "grad_norm": 0.22492443025112152, "kl": 0.0332489013671875, "learning_rate": 2.998209206012151e-06, "loss": 0.0044, "reward": 0.4166666679084301, "reward_std": 0.23899272084236145, "rewards/accuracy_reward": 0.4166666679084301, "rewards/format_reward": 0.0, "step": 171 }, { "clip_ratio": 0.0, "completion_length": 534.4583511352539, "epoch": 0.11466666666666667, "grad_norm": 0.26974058151245117, "kl": 0.04119873046875, "learning_rate": 2.9980346315868857e-06, "loss": -0.0108, "reward": 0.1666666679084301, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 172 }, { "clip_ratio": 0.0, "completion_length": 463.0000228881836, "epoch": 0.11533333333333333, "grad_norm": 0.30081912875175476, "kl": 0.031036376953125, "learning_rate": 2.997851944677384e-06, "loss": 0.0381, "reward": 0.2916666679084301, "reward_std": 0.37592336535453796, "rewards/accuracy_reward": 0.2916666679084301, "rewards/format_reward": 0.0, "step": 173 }, { "clip_ratio": 0.0, "completion_length": 510.91668701171875, "epoch": 0.116, "grad_norm": 0.2714994549751282, "kl": 0.028289794921875, "learning_rate": 2.9976611462729716e-06, "loss": -0.0032, "reward": 0.4583333432674408, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.4583333432674408, "rewards/format_reward": 0.0, "step": 174 }, { "clip_ratio": 0.0, "completion_length": 617.6250228881836, "epoch": 0.11666666666666667, "grad_norm": 0.2659034729003906, "kl": 0.039306640625, "learning_rate": 2.9974622374069026e-06, "loss": 0.0236, "reward": 0.1666666679084301, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 175 }, { "clip_ratio": 0.0, "completion_length": 541.5000152587891, "epoch": 0.11733333333333333, "grad_norm": 0.24003171920776367, "kl": 0.02667236328125, "learning_rate": 2.997255219156351e-06, "loss": 0.0112, "reward": 0.2083333395421505, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 176 }, { "clip_ratio": 0.0, "completion_length": 613.2083435058594, "epoch": 0.118, "grad_norm": 0.23878318071365356, "kl": 0.048614501953125, "learning_rate": 2.9970400926424076e-06, "loss": 0.1285, "reward": 0.2083333358168602, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.2083333358168602, "rewards/format_reward": 0.0, "step": 177 }, { "clip_ratio": 0.0, "completion_length": 487.62501525878906, "epoch": 0.11866666666666667, "grad_norm": 0.3370458483695984, "kl": 0.05621337890625, "learning_rate": 2.996816859030072e-06, "loss": 0.029, "reward": 0.3750000074505806, "reward_std": 0.4701542407274246, "rewards/accuracy_reward": 0.3750000074505806, "rewards/format_reward": 0.0, "step": 178 }, { "clip_ratio": 0.0, "completion_length": 387.87500762939453, "epoch": 0.11933333333333333, "grad_norm": 0.334623783826828, "kl": 0.03936767578125, "learning_rate": 2.9965855195282486e-06, "loss": -0.0523, "reward": 0.5000000186264515, "reward_std": 0.46232306212186813, "rewards/accuracy_reward": 0.5000000186264515, "rewards/format_reward": 0.0, "step": 179 }, { "clip_ratio": 0.0, "completion_length": 589.5416870117188, "epoch": 0.12, "grad_norm": 0.27473437786102295, "kl": 0.04876708984375, "learning_rate": 2.9963460753897363e-06, "loss": 0.0344, "reward": 0.3333333395421505, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.3333333395421505, "rewards/format_reward": 0.0, "step": 180 }, { "clip_ratio": 0.0, "completion_length": 616.0000076293945, "epoch": 0.12066666666666667, "grad_norm": 0.28742659091949463, "kl": 0.06390380859375, "learning_rate": 2.9960985279112264e-06, "loss": 0.0166, "reward": 0.1666666716337204, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 181 }, { "clip_ratio": 0.0, "completion_length": 673.1250152587891, "epoch": 0.12133333333333333, "grad_norm": 0.6071180701255798, "kl": 0.142120361328125, "learning_rate": 2.9958428784332913e-06, "loss": 0.0387, "reward": 0.2083333395421505, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 182 }, { "clip_ratio": 0.0, "completion_length": 469.04168701171875, "epoch": 0.122, "grad_norm": 0.24891294538974762, "kl": 0.031219482421875, "learning_rate": 2.9955791283403805e-06, "loss": 0.0342, "reward": 0.5833333432674408, "reward_std": 0.3872983753681183, "rewards/accuracy_reward": 0.5833333432674408, "rewards/format_reward": 0.0, "step": 183 }, { "clip_ratio": 0.0, "completion_length": 624.2083435058594, "epoch": 0.12266666666666666, "grad_norm": 0.39490076899528503, "kl": 0.08837890625, "learning_rate": 2.995307279060811e-06, "loss": -0.0516, "reward": 0.2500000037252903, "reward_std": 0.44311685860157013, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.0, "step": 184 }, { "clip_ratio": 0.0, "completion_length": 579.6666793823242, "epoch": 0.12333333333333334, "grad_norm": 0.2962397038936615, "kl": 0.0604248046875, "learning_rate": 2.995027332066759e-06, "loss": 0.0194, "reward": 0.2500000037252903, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.0, "step": 185 }, { "clip_ratio": 0.0, "completion_length": 631.5833587646484, "epoch": 0.124, "grad_norm": 0.2961445152759552, "kl": 0.067626953125, "learning_rate": 2.9947392888742567e-06, "loss": 0.1448, "reward": 0.2500000037252903, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.0, "step": 186 }, { "clip_ratio": 0.0, "completion_length": 536.4166717529297, "epoch": 0.12466666666666666, "grad_norm": 0.34972691535949707, "kl": 0.046966552734375, "learning_rate": 2.9944431510431775e-06, "loss": 0.0855, "reward": 0.4583333544433117, "reward_std": 0.306186206638813, "rewards/accuracy_reward": 0.4583333544433117, "rewards/format_reward": 0.0, "step": 187 }, { "clip_ratio": 0.0, "completion_length": 689.4583587646484, "epoch": 0.12533333333333332, "grad_norm": 0.3155274987220764, "kl": 0.0577239990234375, "learning_rate": 2.994138920177231e-06, "loss": 0.0593, "reward": 0.4166666753590107, "reward_std": 0.4971916377544403, "rewards/accuracy_reward": 0.4166666753590107, "rewards/format_reward": 0.0, "step": 188 }, { "clip_ratio": 0.0, "completion_length": 720.6666870117188, "epoch": 0.126, "grad_norm": 0.11947009712457657, "kl": 0.053955078125, "learning_rate": 2.9938265979239552e-06, "loss": 0.0018, "reward": 0.3333333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 189 }, { "clip_ratio": 0.0, "completion_length": 855.5416870117188, "epoch": 0.12666666666666668, "grad_norm": 0.2588626444339752, "kl": 0.0826416015625, "learning_rate": 2.9935061859747068e-06, "loss": 0.0948, "reward": 0.2916666753590107, "reward_std": 0.4893604442477226, "rewards/accuracy_reward": 0.2916666753590107, "rewards/format_reward": 0.0, "step": 190 }, { "clip_ratio": 0.0, "completion_length": 497.6250228881836, "epoch": 0.12733333333333333, "grad_norm": 0.266564279794693, "kl": 0.064483642578125, "learning_rate": 2.993177686064649e-06, "loss": -0.1047, "reward": 0.3333333432674408, "reward_std": 0.3872983753681183, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 191 }, { "clip_ratio": 0.0, "completion_length": 717.7500228881836, "epoch": 0.128, "grad_norm": 0.2919977605342865, "kl": 0.072418212890625, "learning_rate": 2.9928410999727467e-06, "loss": -0.0338, "reward": 0.1666666716337204, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 192 }, { "clip_ratio": 0.0, "completion_length": 612.5416946411133, "epoch": 0.12866666666666668, "grad_norm": 0.35634878277778625, "kl": 0.07061767578125, "learning_rate": 2.9924964295217547e-06, "loss": -0.0239, "reward": 0.4166666828095913, "reward_std": 0.46232305467128754, "rewards/accuracy_reward": 0.4166666828095913, "rewards/format_reward": 0.0, "step": 193 }, { "clip_ratio": 0.0, "completion_length": 494.1666717529297, "epoch": 0.12933333333333333, "grad_norm": 0.3264985680580139, "kl": 0.09039306640625, "learning_rate": 2.9921436765782077e-06, "loss": 0.1698, "reward": 0.3333333432674408, "reward_std": 0.46232306212186813, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 194 }, { "clip_ratio": 0.0, "completion_length": 523.9166717529297, "epoch": 0.13, "grad_norm": 0.4355641007423401, "kl": 0.11468505859375, "learning_rate": 2.99178284305241e-06, "loss": 0.124, "reward": 0.5416666865348816, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.5416666865348816, "rewards/format_reward": 0.0, "step": 195 }, { "clip_ratio": 0.0, "completion_length": 945.5833587646484, "epoch": 0.13066666666666665, "grad_norm": 0.1617182195186615, "kl": 0.11669921875, "learning_rate": 2.9914139308984264e-06, "loss": 0.0778, "reward": 0.1250000037252903, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 196 }, { "clip_ratio": 0.0, "completion_length": 560.2916870117188, "epoch": 0.13133333333333333, "grad_norm": 0.2590388357639313, "kl": 0.0908203125, "learning_rate": 2.991036942114071e-06, "loss": 0.1905, "reward": 0.5416666716337204, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.5416666716337204, "rewards/format_reward": 0.0, "step": 197 }, { "clip_ratio": 0.0, "completion_length": 793.1666870117188, "epoch": 0.132, "grad_norm": 0.22916264832019806, "kl": 0.190673828125, "learning_rate": 2.9906518787408948e-06, "loss": -0.0262, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 198 }, { "clip_ratio": 0.0, "completion_length": 715.6250152587891, "epoch": 0.13266666666666665, "grad_norm": 0.17466306686401367, "kl": 0.100982666015625, "learning_rate": 2.9902587428641774e-06, "loss": 0.0012, "reward": 0.2083333395421505, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 199 }, { "clip_ratio": 0.0, "completion_length": 730.0833511352539, "epoch": 0.13333333333333333, "grad_norm": 0.18963488936424255, "kl": 0.12872314453125, "learning_rate": 2.989857536612915e-06, "loss": 0.0591, "reward": 0.291666679084301, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.291666679084301, "rewards/format_reward": 0.0, "step": 200 }, { "clip_ratio": 0.0, "completion_length": 706.2500152587891, "epoch": 0.134, "grad_norm": 0.2724378705024719, "kl": 0.224609375, "learning_rate": 2.9894482621598055e-06, "loss": 0.0418, "reward": 0.2083333395421505, "reward_std": 0.3602609932422638, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 201 }, { "clip_ratio": 0.0, "completion_length": 783.4583587646484, "epoch": 0.13466666666666666, "grad_norm": 0.2419533133506775, "kl": 0.18341064453125, "learning_rate": 2.989030921721243e-06, "loss": 0.0664, "reward": 0.3750000037252903, "reward_std": 0.5050228163599968, "rewards/accuracy_reward": 0.3750000037252903, "rewards/format_reward": 0.0, "step": 202 }, { "clip_ratio": 0.0, "completion_length": 581.1666717529297, "epoch": 0.13533333333333333, "grad_norm": 0.3563886880874634, "kl": 0.14739990234375, "learning_rate": 2.9886055175572995e-06, "loss": 0.1854, "reward": 0.5833333544433117, "reward_std": 0.44311685860157013, "rewards/accuracy_reward": 0.5833333544433117, "rewards/format_reward": 0.0, "step": 203 }, { "clip_ratio": 0.0, "completion_length": 629.9166870117188, "epoch": 0.136, "grad_norm": 0.3548639714717865, "kl": 0.28515625, "learning_rate": 2.988172051971717e-06, "loss": 0.16, "reward": 0.4166666753590107, "reward_std": 0.4971916377544403, "rewards/accuracy_reward": 0.4166666753590107, "rewards/format_reward": 0.0, "step": 204 }, { "clip_ratio": 0.0, "completion_length": 762.7500152587891, "epoch": 0.13666666666666666, "grad_norm": 0.28060197830200195, "kl": 0.1829833984375, "learning_rate": 2.9877305273118924e-06, "loss": 0.0668, "reward": 0.3333333432674408, "reward_std": 0.23899272084236145, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 205 }, { "clip_ratio": 0.0, "completion_length": 528.8333587646484, "epoch": 0.13733333333333334, "grad_norm": 0.19735652208328247, "kl": 0.20172119140625, "learning_rate": 2.9872809459688676e-06, "loss": -0.0034, "reward": 0.375, "reward_std": 0.1369306445121765, "rewards/accuracy_reward": 0.375, "rewards/format_reward": 0.0, "step": 206 }, { "clip_ratio": 0.0, "completion_length": 676.1250152587891, "epoch": 0.138, "grad_norm": 0.294001042842865, "kl": 0.1805419921875, "learning_rate": 2.9868233103773125e-06, "loss": 0.1066, "reward": 0.3333333395421505, "reward_std": 0.4971916377544403, "rewards/accuracy_reward": 0.3333333395421505, "rewards/format_reward": 0.0, "step": 207 }, { "clip_ratio": 0.0, "completion_length": 613.8333587646484, "epoch": 0.13866666666666666, "grad_norm": 0.3952152132987976, "kl": 0.261474609375, "learning_rate": 2.986357623015516e-06, "loss": 0.0959, "reward": 0.416666679084301, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.416666679084301, "rewards/format_reward": 0.0, "step": 208 }, { "clip_ratio": 0.0, "completion_length": 702.0000152587891, "epoch": 0.13933333333333334, "grad_norm": 0.2392692267894745, "kl": 0.3419189453125, "learning_rate": 2.9858838864053695e-06, "loss": 0.183, "reward": 0.3333333469927311, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.3333333469927311, "rewards/format_reward": 0.0, "step": 209 }, { "clip_ratio": 0.0, "completion_length": 583.7916870117188, "epoch": 0.14, "grad_norm": 0.24478426575660706, "kl": 0.227783203125, "learning_rate": 2.9854021031123555e-06, "loss": 0.1094, "reward": 0.4583333395421505, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.4583333395421505, "rewards/format_reward": 0.0, "step": 210 }, { "clip_ratio": 0.0, "completion_length": 681.5000228881836, "epoch": 0.14066666666666666, "grad_norm": 0.3257905840873718, "kl": 0.266845703125, "learning_rate": 2.984912275745532e-06, "loss": -0.0282, "reward": 0.3750000111758709, "reward_std": 0.4893604442477226, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 211 }, { "clip_ratio": 0.0, "completion_length": 634.3333549499512, "epoch": 0.14133333333333334, "grad_norm": 0.2364731878042221, "kl": 0.24554443359375, "learning_rate": 2.984414406957518e-06, "loss": 0.046, "reward": 0.5000000111758709, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.5000000111758709, "rewards/format_reward": 0.0, "step": 212 }, { "clip_ratio": 0.0, "completion_length": 535.7500152587891, "epoch": 0.142, "grad_norm": 0.33644890785217285, "kl": 0.192626953125, "learning_rate": 2.9839084994444826e-06, "loss": 0.0713, "reward": 0.2916666753590107, "reward_std": 0.4893604442477226, "rewards/accuracy_reward": 0.2916666753590107, "rewards/format_reward": 0.0, "step": 213 }, { "clip_ratio": 0.0, "completion_length": 534.3333511352539, "epoch": 0.14266666666666666, "grad_norm": 0.38609713315963745, "kl": 0.095550537109375, "learning_rate": 2.983394555946126e-06, "loss": -0.038, "reward": 0.2916666716337204, "reward_std": 0.4701542481780052, "rewards/accuracy_reward": 0.2916666716337204, "rewards/format_reward": 0.0, "step": 214 }, { "clip_ratio": 0.0, "completion_length": 654.8333511352539, "epoch": 0.14333333333333334, "grad_norm": 0.2982656955718994, "kl": 0.194305419921875, "learning_rate": 2.9828725792456675e-06, "loss": -0.042, "reward": 0.2500000074505806, "reward_std": 0.3872983753681183, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 215 }, { "clip_ratio": 0.0, "completion_length": 625.6250228881836, "epoch": 0.144, "grad_norm": 0.15318594872951508, "kl": 0.24530029296875, "learning_rate": 2.9823425721698293e-06, "loss": 0.048, "reward": 0.2916666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.2916666679084301, "rewards/format_reward": 0.0, "step": 216 }, { "clip_ratio": 0.0, "completion_length": 559.7500114440918, "epoch": 0.14466666666666667, "grad_norm": 0.18845723569393158, "kl": 0.1893310546875, "learning_rate": 2.9818045375888215e-06, "loss": -0.0002, "reward": 0.4583333395421505, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.4583333395421505, "rewards/format_reward": 0.0, "step": 217 }, { "clip_ratio": 0.0, "completion_length": 691.1666870117188, "epoch": 0.14533333333333334, "grad_norm": 0.32294532656669617, "kl": 0.12158203125, "learning_rate": 2.9812584784163257e-06, "loss": 0.145, "reward": 0.3750000074505806, "reward_std": 0.39512956142425537, "rewards/accuracy_reward": 0.3750000074505806, "rewards/format_reward": 0.0, "step": 218 }, { "clip_ratio": 0.0, "completion_length": 596.8333435058594, "epoch": 0.146, "grad_norm": 0.3616347312927246, "kl": 0.2230224609375, "learning_rate": 2.980704397609482e-06, "loss": 0.2134, "reward": 0.3750000111758709, "reward_std": 0.4893604516983032, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 219 }, { "clip_ratio": 0.0, "completion_length": 649.8750228881836, "epoch": 0.14666666666666667, "grad_norm": 0.2796820402145386, "kl": 0.228271484375, "learning_rate": 2.980142298168869e-06, "loss": 0.0352, "reward": 0.2083333358168602, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.2083333358168602, "rewards/format_reward": 0.0, "step": 220 }, { "clip_ratio": 0.0, "completion_length": 607.9583587646484, "epoch": 0.14733333333333334, "grad_norm": 0.3202641010284424, "kl": 0.10101318359375, "learning_rate": 2.97957218313849e-06, "loss": -0.0205, "reward": 0.3750000149011612, "reward_std": 0.43528568744659424, "rewards/accuracy_reward": 0.3750000149011612, "rewards/format_reward": 0.0, "step": 221 }, { "clip_ratio": 0.0, "completion_length": 527.2083511352539, "epoch": 0.148, "grad_norm": 0.19183184206485748, "kl": 0.08306884765625, "learning_rate": 2.9789940556057576e-06, "loss": 0.0243, "reward": 0.4166666679084301, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.4166666679084301, "rewards/format_reward": 0.0, "step": 222 }, { "clip_ratio": 0.0, "completion_length": 428.5416717529297, "epoch": 0.14866666666666667, "grad_norm": 0.29112136363983154, "kl": 0.203369140625, "learning_rate": 2.978407918701473e-06, "loss": 0.0244, "reward": 0.4583333395421505, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.4583333395421505, "rewards/format_reward": 0.0, "step": 223 }, { "clip_ratio": 0.0, "completion_length": 620.2916793823242, "epoch": 0.14933333333333335, "grad_norm": 0.2713090777397156, "kl": 0.14678955078125, "learning_rate": 2.9778137755998135e-06, "loss": -0.0797, "reward": 0.0833333358168602, "reward_std": 0.20412415266036987, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 224 }, { "clip_ratio": 0.0, "completion_length": 388.6666717529297, "epoch": 0.15, "grad_norm": 0.20526248216629028, "kl": 0.0950927734375, "learning_rate": 2.9772116295183124e-06, "loss": 0.0406, "reward": 0.7500000111758709, "reward_std": 0.20412415266036987, "rewards/accuracy_reward": 0.7500000111758709, "rewards/format_reward": 0.0, "step": 225 }, { "clip_ratio": 0.0, "completion_length": 391.4583435058594, "epoch": 0.15066666666666667, "grad_norm": 0.3988746106624603, "kl": 0.13427734375, "learning_rate": 2.9766014837178418e-06, "loss": 0.0308, "reward": 0.3750000111758709, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 226 }, { "clip_ratio": 0.0, "completion_length": 529.1250228881836, "epoch": 0.15133333333333332, "grad_norm": 0.7330386638641357, "kl": 0.3408203125, "learning_rate": 2.975983341502598e-06, "loss": 0.1046, "reward": 0.6666666716337204, "reward_std": 0.40296074748039246, "rewards/accuracy_reward": 0.6666666716337204, "rewards/format_reward": 0.0, "step": 227 }, { "clip_ratio": 0.0, "completion_length": 472.75000762939453, "epoch": 0.152, "grad_norm": 1.3451703786849976, "kl": 0.5697021484375, "learning_rate": 2.975357206220079e-06, "loss": 0.0395, "reward": 0.6666666716337204, "reward_std": 0.40296074748039246, "rewards/accuracy_reward": 0.6666666716337204, "rewards/format_reward": 0.0, "step": 228 }, { "clip_ratio": 0.0, "completion_length": 572.3750190734863, "epoch": 0.15266666666666667, "grad_norm": 0.3637828826904297, "kl": 0.28533935546875, "learning_rate": 2.974723081261069e-06, "loss": 0.066, "reward": 0.291666679084301, "reward_std": 0.3061862140893936, "rewards/accuracy_reward": 0.291666679084301, "rewards/format_reward": 0.0, "step": 229 }, { "clip_ratio": 0.0, "completion_length": 687.3750152587891, "epoch": 0.15333333333333332, "grad_norm": 2.504993438720703, "kl": 2.255615234375, "learning_rate": 2.97408097005962e-06, "loss": 0.1177, "reward": 0.2083333395421505, "reward_std": 0.3602609932422638, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 230 }, { "clip_ratio": 0.0, "completion_length": 563.6250152587891, "epoch": 0.154, "grad_norm": 1.765861988067627, "kl": 0.901885986328125, "learning_rate": 2.9734308760930334e-06, "loss": 0.0594, "reward": 0.4166666716337204, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.4166666716337204, "rewards/format_reward": 0.0, "step": 231 }, { "clip_ratio": 0.0, "completion_length": 536.2083435058594, "epoch": 0.15466666666666667, "grad_norm": 0.8455492258071899, "kl": 0.624267578125, "learning_rate": 2.9727728028818388e-06, "loss": 0.0901, "reward": 0.2083333358168602, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.2083333358168602, "rewards/format_reward": 0.0, "step": 232 }, { "clip_ratio": 0.0, "completion_length": 592.0416793823242, "epoch": 0.15533333333333332, "grad_norm": 2.750101089477539, "kl": 0.8271484375, "learning_rate": 2.9721067539897793e-06, "loss": 0.1518, "reward": 0.3333333432674408, "reward_std": 0.23899272084236145, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 233 }, { "clip_ratio": 0.0, "completion_length": 615.1250152587891, "epoch": 0.156, "grad_norm": 1.351430058479309, "kl": 0.7879638671875, "learning_rate": 2.9714327330237873e-06, "loss": 0.1091, "reward": 0.2500000037252903, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.0, "step": 234 }, { "clip_ratio": 0.0, "completion_length": 688.0833435058594, "epoch": 0.15666666666666668, "grad_norm": 5.44602632522583, "kl": 4.36224365234375, "learning_rate": 2.970750743633969e-06, "loss": 0.2413, "reward": 0.4583333507180214, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.4583333507180214, "rewards/format_reward": 0.0, "step": 235 }, { "clip_ratio": 0.0, "completion_length": 580.4166870117188, "epoch": 0.15733333333333333, "grad_norm": 1.4070898294448853, "kl": 0.87451171875, "learning_rate": 2.970060789513582e-06, "loss": 0.1169, "reward": 0.4166666679084301, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.4166666679084301, "rewards/format_reward": 0.0, "step": 236 }, { "clip_ratio": 0.0, "completion_length": 542.5833511352539, "epoch": 0.158, "grad_norm": 3.6622698307037354, "kl": 0.365234375, "learning_rate": 2.969362874399016e-06, "loss": 0.0907, "reward": 0.5000000074505806, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.5000000074505806, "rewards/format_reward": 0.0, "step": 237 }, { "clip_ratio": 0.0, "completion_length": 613.5, "epoch": 0.15866666666666668, "grad_norm": 2.463667392730713, "kl": 0.71954345703125, "learning_rate": 2.968657002069774e-06, "loss": 0.1246, "reward": 0.2500000074505806, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 238 }, { "clip_ratio": 0.0, "completion_length": 612.2916870117188, "epoch": 0.15933333333333333, "grad_norm": 1.7052408456802368, "kl": 1.3671875, "learning_rate": 2.96794317634845e-06, "loss": 0.062, "reward": 0.4166666716337204, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.4166666716337204, "rewards/format_reward": 0.0, "step": 239 }, { "clip_ratio": 0.0, "completion_length": 812.1666870117188, "epoch": 0.16, "grad_norm": 0.7843796014785767, "kl": 0.83135986328125, "learning_rate": 2.9672214011007086e-06, "loss": 0.0799, "reward": 0.2916666753590107, "reward_std": 0.43528566509485245, "rewards/accuracy_reward": 0.2916666753590107, "rewards/format_reward": 0.0, "step": 240 }, { "clip_ratio": 0.0, "completion_length": 683.0416793823242, "epoch": 0.16066666666666668, "grad_norm": 12.39732551574707, "kl": 1.48681640625, "learning_rate": 2.9664916802352644e-06, "loss": 0.1615, "reward": 0.5000000223517418, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.5000000223517418, "rewards/format_reward": 0.0, "step": 241 }, { "clip_ratio": 0.0, "completion_length": 717.7916717529297, "epoch": 0.16133333333333333, "grad_norm": 0.514663577079773, "kl": 0.181396484375, "learning_rate": 2.965754017703862e-06, "loss": 0.0635, "reward": 0.416666679084301, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.416666679084301, "rewards/format_reward": 0.0, "step": 242 }, { "clip_ratio": 0.0, "completion_length": 756.2916870117188, "epoch": 0.162, "grad_norm": 1.4085949659347534, "kl": 1.06640625, "learning_rate": 2.965008417501252e-06, "loss": 0.0999, "reward": 0.2500000037252903, "reward_std": 0.3680921792984009, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.0, "step": 243 }, { "clip_ratio": 0.0, "completion_length": 587.8750152587891, "epoch": 0.16266666666666665, "grad_norm": 2.3212459087371826, "kl": 0.42724609375, "learning_rate": 2.9642548836651712e-06, "loss": 0.054, "reward": 0.2500000037252903, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.0, "step": 244 }, { "clip_ratio": 0.0, "completion_length": 730.5000152587891, "epoch": 0.16333333333333333, "grad_norm": 2.0084919929504395, "kl": 0.509765625, "learning_rate": 2.9634934202763216e-06, "loss": 0.0719, "reward": 0.2083333395421505, "reward_std": 0.3602609932422638, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 245 }, { "clip_ratio": 0.0, "completion_length": 636.8333587646484, "epoch": 0.164, "grad_norm": 0.7770030498504639, "kl": 0.21588134765625, "learning_rate": 2.962724031458345e-06, "loss": 0.0965, "reward": 0.4583333469927311, "reward_std": 0.4893604442477226, "rewards/accuracy_reward": 0.4583333469927311, "rewards/format_reward": 0.0, "step": 246 }, { "clip_ratio": 0.0, "completion_length": 496.9166717529297, "epoch": 0.16466666666666666, "grad_norm": 0.3213016390800476, "kl": 0.2322998046875, "learning_rate": 2.961946721377804e-06, "loss": 0.0324, "reward": 0.5000000223517418, "reward_std": 0.3332236111164093, "rewards/accuracy_reward": 0.5000000223517418, "rewards/format_reward": 0.0, "step": 247 }, { "clip_ratio": 0.0, "completion_length": 482.9583435058594, "epoch": 0.16533333333333333, "grad_norm": 0.3044670522212982, "kl": 0.167999267578125, "learning_rate": 2.9611614942441577e-06, "loss": 0.0858, "reward": 0.5833333544433117, "reward_std": 0.44311685860157013, "rewards/accuracy_reward": 0.5833333544433117, "rewards/format_reward": 0.0, "step": 248 }, { "clip_ratio": 0.0, "completion_length": 582.0416946411133, "epoch": 0.166, "grad_norm": 0.738615095615387, "kl": 0.9375, "learning_rate": 2.9603683543097407e-06, "loss": 0.0902, "reward": 0.5833333432674408, "reward_std": 0.3872983753681183, "rewards/accuracy_reward": 0.5833333432674408, "rewards/format_reward": 0.0, "step": 249 }, { "clip_ratio": 0.0, "completion_length": 565.6250228881836, "epoch": 0.16666666666666666, "grad_norm": 3.3314404487609863, "kl": 2.6907958984375, "learning_rate": 2.959567305869736e-06, "loss": 0.1461, "reward": 0.3333333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 250 }, { "clip_ratio": 0.0, "completion_length": 753.9583587646484, "epoch": 0.16733333333333333, "grad_norm": 0.5914206504821777, "kl": 0.948455810546875, "learning_rate": 2.9587583532621567e-06, "loss": 0.062, "reward": 0.3333333395421505, "reward_std": 0.4971916303038597, "rewards/accuracy_reward": 0.3333333395421505, "rewards/format_reward": 0.0, "step": 251 }, { "clip_ratio": 0.0, "completion_length": 691.4166870117188, "epoch": 0.168, "grad_norm": 0.5690410137176514, "kl": 1.31817626953125, "learning_rate": 2.95794150086782e-06, "loss": 0.0764, "reward": 0.416666679084301, "reward_std": 0.4431168660521507, "rewards/accuracy_reward": 0.416666679084301, "rewards/format_reward": 0.0, "step": 252 }, { "clip_ratio": 0.0, "completion_length": 742.5416870117188, "epoch": 0.16866666666666666, "grad_norm": 0.6222215890884399, "kl": 0.569183349609375, "learning_rate": 2.957116753110322e-06, "loss": 0.0411, "reward": 0.1666666716337204, "reward_std": 0.3332236111164093, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 253 }, { "clip_ratio": 0.0, "completion_length": 716.2916946411133, "epoch": 0.16933333333333334, "grad_norm": 1.251683235168457, "kl": 1.359771728515625, "learning_rate": 2.956284114456018e-06, "loss": 0.0764, "reward": 0.291666679084301, "reward_std": 0.3061862140893936, "rewards/accuracy_reward": 0.291666679084301, "rewards/format_reward": 0.0, "step": 254 }, { "clip_ratio": 0.0, "completion_length": 562.5416946411133, "epoch": 0.17, "grad_norm": 0.8058119416236877, "kl": 0.625244140625, "learning_rate": 2.9554435894139947e-06, "loss": 0.0637, "reward": 0.4166666716337204, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.4166666716337204, "rewards/format_reward": 0.0, "step": 255 }, { "clip_ratio": 0.0, "completion_length": 463.25000762939453, "epoch": 0.17066666666666666, "grad_norm": 1.6968247890472412, "kl": 0.23175048828125, "learning_rate": 2.9545951825360466e-06, "loss": 0.034, "reward": 0.4583333507180214, "reward_std": 0.4893604442477226, "rewards/accuracy_reward": 0.4583333507180214, "rewards/format_reward": 0.0, "step": 256 }, { "clip_ratio": 0.0, "completion_length": 590.5000152587891, "epoch": 0.17133333333333334, "grad_norm": 0.43418627977371216, "kl": 0.4466552734375, "learning_rate": 2.9537388984166518e-06, "loss": 0.0167, "reward": 0.4166666716337204, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.4166666716337204, "rewards/format_reward": 0.0, "step": 257 }, { "clip_ratio": 0.0, "completion_length": 676.2916870117188, "epoch": 0.172, "grad_norm": 2.2851316928863525, "kl": 0.513397216796875, "learning_rate": 2.9528747416929465e-06, "loss": 0.1276, "reward": 0.3750000111758709, "reward_std": 0.3410547822713852, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 258 }, { "clip_ratio": 0.0, "completion_length": 654.7083435058594, "epoch": 0.17266666666666666, "grad_norm": 0.430602103471756, "kl": 0.29437255859375, "learning_rate": 2.952002717044702e-06, "loss": 0.0454, "reward": 0.416666679084301, "reward_std": 0.3872983753681183, "rewards/accuracy_reward": 0.416666679084301, "rewards/format_reward": 0.0, "step": 259 }, { "clip_ratio": 0.0, "completion_length": 578.7916793823242, "epoch": 0.17333333333333334, "grad_norm": 0.3225778341293335, "kl": 0.136199951171875, "learning_rate": 2.951122829194296e-06, "loss": 0.0149, "reward": 0.5000000074505806, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.5000000074505806, "rewards/format_reward": 0.0, "step": 260 }, { "clip_ratio": 0.0, "completion_length": 661.0000152587891, "epoch": 0.174, "grad_norm": 0.7602376341819763, "kl": 0.880859375, "learning_rate": 2.950235082906689e-06, "loss": 0.162, "reward": 0.2083333395421505, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 261 }, { "clip_ratio": 0.0, "completion_length": 444.7916793823242, "epoch": 0.17466666666666666, "grad_norm": 1.4374006986618042, "kl": 1.181396484375, "learning_rate": 2.9493394829893994e-06, "loss": 0.2146, "reward": 0.6250000111758709, "reward_std": 0.3602609932422638, "rewards/accuracy_reward": 0.6250000111758709, "rewards/format_reward": 0.0, "step": 262 }, { "clip_ratio": 0.0, "completion_length": 593.3333435058594, "epoch": 0.17533333333333334, "grad_norm": 0.1744527816772461, "kl": 0.1961669921875, "learning_rate": 2.948436034292474e-06, "loss": 0.0484, "reward": 0.2916666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.2916666679084301, "rewards/format_reward": 0.0, "step": 263 }, { "clip_ratio": 0.0, "completion_length": 625.5833587646484, "epoch": 0.176, "grad_norm": 5.297957897186279, "kl": 2.69818115234375, "learning_rate": 2.9475247417084673e-06, "loss": 0.202, "reward": 0.2500000074505806, "reward_std": 0.3872983753681183, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 264 }, { "clip_ratio": 0.0, "completion_length": 835.5833435058594, "epoch": 0.17666666666666667, "grad_norm": 0.9309121370315552, "kl": 1.130615234375, "learning_rate": 2.9466056101724086e-06, "loss": 0.0703, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 265 }, { "clip_ratio": 0.0, "completion_length": 471.1250114440918, "epoch": 0.17733333333333334, "grad_norm": 0.33158740401268005, "kl": 0.2559814453125, "learning_rate": 2.9456786446617797e-06, "loss": -0.0028, "reward": 0.4166666865348816, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.4166666865348816, "rewards/format_reward": 0.0, "step": 266 }, { "clip_ratio": 0.0, "completion_length": 482.9583435058594, "epoch": 0.178, "grad_norm": 0.22709830105304718, "kl": 0.042816162109375, "learning_rate": 2.9447438501964875e-06, "loss": 0.0212, "reward": 0.6666666828095913, "reward_std": 0.3332236111164093, "rewards/accuracy_reward": 0.6666666828095913, "rewards/format_reward": 0.0, "step": 267 }, { "clip_ratio": 0.0, "completion_length": 447.75001525878906, "epoch": 0.17866666666666667, "grad_norm": 0.2314523607492447, "kl": 0.128692626953125, "learning_rate": 2.9438012318388337e-06, "loss": -0.0093, "reward": 0.666666679084301, "reward_std": 0.3680921792984009, "rewards/accuracy_reward": 0.666666679084301, "rewards/format_reward": 0.0, "step": 268 }, { "clip_ratio": 0.0, "completion_length": 678.0416870117188, "epoch": 0.17933333333333334, "grad_norm": 0.3418677747249603, "kl": 0.13385009765625, "learning_rate": 2.9428507946934932e-06, "loss": -0.0284, "reward": 0.3333333432674408, "reward_std": 0.23899272084236145, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 269 }, { "clip_ratio": 0.0, "completion_length": 658.1666946411133, "epoch": 0.18, "grad_norm": 0.27784982323646545, "kl": 0.12457275390625, "learning_rate": 2.9418925439074784e-06, "loss": 0.0181, "reward": 0.0833333358168602, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 270 }, { "clip_ratio": 0.0, "completion_length": 581.8333435058594, "epoch": 0.18066666666666667, "grad_norm": 0.35795858502388, "kl": 0.1773681640625, "learning_rate": 2.9409264846701195e-06, "loss": 0.0749, "reward": 0.2916666753590107, "reward_std": 0.43528567254543304, "rewards/accuracy_reward": 0.2916666753590107, "rewards/format_reward": 0.0, "step": 271 }, { "clip_ratio": 0.0, "completion_length": 690.4166946411133, "epoch": 0.18133333333333335, "grad_norm": 0.49630066752433777, "kl": 0.206787109375, "learning_rate": 2.9399526222130314e-06, "loss": 0.0728, "reward": 0.2916666753590107, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.2916666753590107, "rewards/format_reward": 0.0, "step": 272 }, { "clip_ratio": 0.0, "completion_length": 545.6666946411133, "epoch": 0.182, "grad_norm": 0.3159043490886688, "kl": 0.10455322265625, "learning_rate": 2.9389709618100864e-06, "loss": 0.0103, "reward": 0.2083333395421505, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 273 }, { "clip_ratio": 0.0, "completion_length": 614.6250305175781, "epoch": 0.18266666666666667, "grad_norm": 0.6409536600112915, "kl": 0.10882568359375, "learning_rate": 2.9379815087773864e-06, "loss": 0.1838, "reward": 0.416666679084301, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.416666679084301, "rewards/format_reward": 0.0, "step": 274 }, { "clip_ratio": 0.0, "completion_length": 540.0000076293945, "epoch": 0.18333333333333332, "grad_norm": 0.24980148673057556, "kl": 0.07843017578125, "learning_rate": 2.9369842684732336e-06, "loss": -0.002, "reward": 0.7500000149011612, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.7500000149011612, "rewards/format_reward": 0.0, "step": 275 }, { "clip_ratio": 0.0, "completion_length": 526.1250152587891, "epoch": 0.184, "grad_norm": 0.4798089265823364, "kl": 0.1322021484375, "learning_rate": 2.9359792462981008e-06, "loss": -0.0213, "reward": 0.541666679084301, "reward_std": 0.5050228238105774, "rewards/accuracy_reward": 0.541666679084301, "rewards/format_reward": 0.0, "step": 276 }, { "clip_ratio": 0.0, "completion_length": 355.6250114440918, "epoch": 0.18466666666666667, "grad_norm": 0.33847761154174805, "kl": 0.06561279296875, "learning_rate": 2.9349664476946045e-06, "loss": -0.025, "reward": 0.4583333395421505, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.4583333395421505, "rewards/format_reward": 0.0, "step": 277 }, { "clip_ratio": 0.0, "completion_length": 591.9166870117188, "epoch": 0.18533333333333332, "grad_norm": 0.39349398016929626, "kl": 0.16796875, "learning_rate": 2.9339458781474724e-06, "loss": 0.077, "reward": 0.3333333358168602, "reward_std": 0.4779854342341423, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 278 }, { "clip_ratio": 0.0, "completion_length": 485.7500114440918, "epoch": 0.186, "grad_norm": 0.7614178657531738, "kl": 0.27197265625, "learning_rate": 2.9329175431835153e-06, "loss": 0.1201, "reward": 0.291666679084301, "reward_std": 0.3061862140893936, "rewards/accuracy_reward": 0.291666679084301, "rewards/format_reward": 0.0, "step": 279 }, { "clip_ratio": 0.0, "completion_length": 569.4166793823242, "epoch": 0.18666666666666668, "grad_norm": 0.2068711221218109, "kl": 0.0802001953125, "learning_rate": 2.9318814483715983e-06, "loss": 0.0264, "reward": 0.3750000037252903, "reward_std": 0.3061862140893936, "rewards/accuracy_reward": 0.3750000037252903, "rewards/format_reward": 0.0, "step": 280 }, { "clip_ratio": 0.0, "completion_length": 765.2917022705078, "epoch": 0.18733333333333332, "grad_norm": 0.35449498891830444, "kl": 0.61865234375, "learning_rate": 2.9308375993226066e-06, "loss": 0.0552, "reward": 0.2500000037252903, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.0, "step": 281 }, { "clip_ratio": 0.0, "completion_length": 445.50001525878906, "epoch": 0.188, "grad_norm": 0.9312465190887451, "kl": 0.5850830078125, "learning_rate": 2.9297860016894203e-06, "loss": 0.0307, "reward": 0.5833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.5833333358168602, "rewards/format_reward": 0.0, "step": 282 }, { "clip_ratio": 0.0, "completion_length": 677.8333587646484, "epoch": 0.18866666666666668, "grad_norm": 0.7988510727882385, "kl": 0.49609375, "learning_rate": 2.9287266611668812e-06, "loss": 0.1594, "reward": 0.3333333395421505, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.3333333395421505, "rewards/format_reward": 0.0, "step": 283 }, { "clip_ratio": 0.0, "completion_length": 664.3750152587891, "epoch": 0.18933333333333333, "grad_norm": 0.4683569073677063, "kl": 0.39300537109375, "learning_rate": 2.9276595834917606e-06, "loss": 0.0462, "reward": 0.2916666753590107, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.2916666753590107, "rewards/format_reward": 0.0, "step": 284 }, { "clip_ratio": 0.0, "completion_length": 721.9166870117188, "epoch": 0.19, "grad_norm": 0.5071645379066467, "kl": 0.666259765625, "learning_rate": 2.9265847744427307e-06, "loss": 0.1273, "reward": 0.416666679084301, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.416666679084301, "rewards/format_reward": 0.0, "step": 285 }, { "clip_ratio": 0.0, "completion_length": 589.0000152587891, "epoch": 0.19066666666666668, "grad_norm": 0.6792252659797668, "kl": 1.31640625, "learning_rate": 2.925502239840332e-06, "loss": 0.0673, "reward": 0.4583333507180214, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.4583333507180214, "rewards/format_reward": 0.0, "step": 286 }, { "clip_ratio": 0.0, "completion_length": 705.6250152587891, "epoch": 0.19133333333333333, "grad_norm": 0.7028434872627258, "kl": 1.84716796875, "learning_rate": 2.924411985546942e-06, "loss": 0.239, "reward": 0.5000000074505806, "reward_std": 0.5320602059364319, "rewards/accuracy_reward": 0.5000000074505806, "rewards/format_reward": 0.0, "step": 287 }, { "clip_ratio": 0.0, "completion_length": 581.7083358764648, "epoch": 0.192, "grad_norm": 1.7310619354248047, "kl": 2.0599365234375, "learning_rate": 2.9233140174667447e-06, "loss": 0.021, "reward": 0.2083333358168602, "reward_std": 0.3410547822713852, "rewards/accuracy_reward": 0.2083333358168602, "rewards/format_reward": 0.0, "step": 288 }, { "clip_ratio": 0.0, "completion_length": 584.9583587646484, "epoch": 0.19266666666666668, "grad_norm": 1.3050038814544678, "kl": 0.534027099609375, "learning_rate": 2.922208341545696e-06, "loss": 0.114, "reward": 0.2500000074505806, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 289 }, { "clip_ratio": 0.0, "completion_length": 631.2083587646484, "epoch": 0.19333333333333333, "grad_norm": 0.8427775502204895, "kl": 1.0106201171875, "learning_rate": 2.921094963771494e-06, "loss": 0.1483, "reward": 0.3333333432674408, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 290 }, { "clip_ratio": 0.0, "completion_length": 751.4166793823242, "epoch": 0.194, "grad_norm": 0.5921525955200195, "kl": 1.18017578125, "learning_rate": 2.9199738901735447e-06, "loss": 0.2112, "reward": 0.1666666716337204, "reward_std": 0.40824829041957855, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 291 }, { "clip_ratio": 0.0, "completion_length": 624.1250228881836, "epoch": 0.19466666666666665, "grad_norm": 1.0096373558044434, "kl": 1.478515625, "learning_rate": 2.9188451268229305e-06, "loss": 0.2291, "reward": 0.5833333544433117, "reward_std": 0.46232306212186813, "rewards/accuracy_reward": 0.5833333544433117, "rewards/format_reward": 0.0, "step": 292 }, { "clip_ratio": 0.0, "completion_length": 701.9583587646484, "epoch": 0.19533333333333333, "grad_norm": 0.8499011397361755, "kl": 1.26226806640625, "learning_rate": 2.917708679832378e-06, "loss": 0.1252, "reward": 0.541666679084301, "reward_std": 0.3061862215399742, "rewards/accuracy_reward": 0.541666679084301, "rewards/format_reward": 0.0, "step": 293 }, { "clip_ratio": 0.0, "completion_length": 673.9166946411133, "epoch": 0.196, "grad_norm": 0.813515305519104, "kl": 0.19677734375, "learning_rate": 2.9165645553562214e-06, "loss": 0.0295, "reward": 0.3750000149011612, "reward_std": 0.43528567999601364, "rewards/accuracy_reward": 0.3750000149011612, "rewards/format_reward": 0.0, "step": 294 }, { "clip_ratio": 0.0, "completion_length": 690.8333587646484, "epoch": 0.19666666666666666, "grad_norm": 3.2097578048706055, "kl": 4.2744140625, "learning_rate": 2.9154127595903755e-06, "loss": 0.3998, "reward": 0.541666679084301, "reward_std": 0.5242290198802948, "rewards/accuracy_reward": 0.541666679084301, "rewards/format_reward": 0.0, "step": 295 }, { "clip_ratio": 0.0, "completion_length": 489.9166717529297, "epoch": 0.19733333333333333, "grad_norm": 1.000464916229248, "kl": 0.397216796875, "learning_rate": 2.914253298772295e-06, "loss": -0.0117, "reward": 0.5416666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.5416666679084301, "rewards/format_reward": 0.0, "step": 296 }, { "clip_ratio": 0.0, "completion_length": 631.0833511352539, "epoch": 0.198, "grad_norm": 1.4472906589508057, "kl": 0.43359375, "learning_rate": 2.913086179180945e-06, "loss": 0.0635, "reward": 0.4583333432674408, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.4583333432674408, "rewards/format_reward": 0.0, "step": 297 }, { "clip_ratio": 0.0, "completion_length": 743.9166870117188, "epoch": 0.19866666666666666, "grad_norm": 0.3217608630657196, "kl": 0.169677734375, "learning_rate": 2.9119114071367674e-06, "loss": 0.0959, "reward": 0.2500000074505806, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 298 }, { "clip_ratio": 0.0, "completion_length": 665.0416793823242, "epoch": 0.19933333333333333, "grad_norm": 0.8076165318489075, "kl": 0.2586669921875, "learning_rate": 2.9107289890016436e-06, "loss": 0.0662, "reward": 0.3750000111758709, "reward_std": 0.3410547897219658, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 299 }, { "clip_ratio": 0.0, "completion_length": 697.0833435058594, "epoch": 0.2, "grad_norm": 0.538249671459198, "kl": 0.425048828125, "learning_rate": 2.9095389311788626e-06, "loss": 0.061, "reward": 0.3333333395421505, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.3333333395421505, "rewards/format_reward": 0.0, "step": 300 }, { "clip_ratio": 0.0, "completion_length": 772.2500152587891, "epoch": 0.20066666666666666, "grad_norm": 0.7322794198989868, "kl": 1.08056640625, "learning_rate": 2.9083412401130855e-06, "loss": 0.1928, "reward": 0.1666666716337204, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 301 }, { "clip_ratio": 0.0, "completion_length": 658.5000076293945, "epoch": 0.20133333333333334, "grad_norm": 2.3514180183410645, "kl": 1.071533203125, "learning_rate": 2.9071359222903105e-06, "loss": 0.1628, "reward": 0.4583333395421505, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.4583333395421505, "rewards/format_reward": 0.0, "step": 302 }, { "clip_ratio": 0.0, "completion_length": 827.2500305175781, "epoch": 0.202, "grad_norm": 2.679455518722534, "kl": 2.572265625, "learning_rate": 2.9059229842378375e-06, "loss": 0.1113, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 303 }, { "clip_ratio": 0.0, "completion_length": 718.6250305175781, "epoch": 0.20266666666666666, "grad_norm": 1.33988618850708, "kl": 1.20068359375, "learning_rate": 2.9047024325242336e-06, "loss": 0.0495, "reward": 0.3333333432674408, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 304 }, { "clip_ratio": 0.0, "completion_length": 668.9583435058594, "epoch": 0.20333333333333334, "grad_norm": 4.550530910491943, "kl": 0.69873046875, "learning_rate": 2.9034742737592973e-06, "loss": 0.0886, "reward": 0.2083333358168602, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.2083333358168602, "rewards/format_reward": 0.0, "step": 305 }, { "clip_ratio": 0.0, "completion_length": 766.9583435058594, "epoch": 0.204, "grad_norm": 3.6896283626556396, "kl": 0.675048828125, "learning_rate": 2.9022385145940218e-06, "loss": 0.223, "reward": 0.3750000111758709, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 306 }, { "clip_ratio": 0.0, "completion_length": 711.0000152587891, "epoch": 0.20466666666666666, "grad_norm": 0.7418251037597656, "kl": 1.0546875, "learning_rate": 2.90099516172056e-06, "loss": 0.2563, "reward": 0.3750000074505806, "reward_std": 0.39512956142425537, "rewards/accuracy_reward": 0.3750000074505806, "rewards/format_reward": 0.0, "step": 307 }, { "clip_ratio": 0.0, "completion_length": 675.4166793823242, "epoch": 0.20533333333333334, "grad_norm": 0.6857995390892029, "kl": 1.2138671875, "learning_rate": 2.899744221872188e-06, "loss": 0.1109, "reward": 0.3333333432674408, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 308 }, { "clip_ratio": 0.0, "completion_length": 560.9583511352539, "epoch": 0.206, "grad_norm": 1.4127880334854126, "kl": 0.7420654296875, "learning_rate": 2.898485701823268e-06, "loss": 0.1798, "reward": 0.3333333432674408, "reward_std": 0.3872983753681183, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 309 }, { "clip_ratio": 0.0, "completion_length": 667.6666717529297, "epoch": 0.20666666666666667, "grad_norm": 2.318650722503662, "kl": 2.41583251953125, "learning_rate": 2.8972196083892137e-06, "loss": 0.183, "reward": 0.4166666753590107, "reward_std": 0.4971916303038597, "rewards/accuracy_reward": 0.4166666753590107, "rewards/format_reward": 0.0, "step": 310 }, { "clip_ratio": 0.0, "completion_length": 739.8750305175781, "epoch": 0.20733333333333334, "grad_norm": 3.0816071033477783, "kl": 3.4765625, "learning_rate": 2.8959459484264492e-06, "loss": 0.2381, "reward": 0.3333333358168602, "reward_std": 0.4779854342341423, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 311 }, { "clip_ratio": 0.0, "completion_length": 526.4583358764648, "epoch": 0.208, "grad_norm": 0.42306697368621826, "kl": 1.1766357421875, "learning_rate": 2.894664728832377e-06, "loss": 0.2013, "reward": 0.6250000149011612, "reward_std": 0.3602609932422638, "rewards/accuracy_reward": 0.6250000149011612, "rewards/format_reward": 0.0, "step": 312 }, { "clip_ratio": 0.0, "completion_length": 617.0000152587891, "epoch": 0.20866666666666667, "grad_norm": 0.9581692814826965, "kl": 1.58892822265625, "learning_rate": 2.8933759565453373e-06, "loss": 0.2669, "reward": 0.4583333432674408, "reward_std": 0.37592335790395737, "rewards/accuracy_reward": 0.4583333432674408, "rewards/format_reward": 0.0, "step": 313 }, { "clip_ratio": 0.0, "completion_length": 656.7500152587891, "epoch": 0.20933333333333334, "grad_norm": 0.6647593975067139, "kl": 1.1923828125, "learning_rate": 2.8920796385445705e-06, "loss": 0.141, "reward": 0.291666679084301, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.291666679084301, "rewards/format_reward": 0.0, "step": 314 }, { "clip_ratio": 0.0, "completion_length": 670.1250152587891, "epoch": 0.21, "grad_norm": 0.4789959192276001, "kl": 0.46337890625, "learning_rate": 2.8907757818501814e-06, "loss": 0.0368, "reward": 0.5000000074505806, "reward_std": 0.5320602059364319, "rewards/accuracy_reward": 0.5000000074505806, "rewards/format_reward": 0.0, "step": 315 }, { "clip_ratio": 0.0, "completion_length": 623.2916946411133, "epoch": 0.21066666666666667, "grad_norm": 0.7020223736763, "kl": 0.52410888671875, "learning_rate": 2.889464393523099e-06, "loss": 0.0939, "reward": 0.416666679084301, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.416666679084301, "rewards/format_reward": 0.0, "step": 316 }, { "clip_ratio": 0.0, "completion_length": 707.6666870117188, "epoch": 0.21133333333333335, "grad_norm": 2.2613768577575684, "kl": 1.004669189453125, "learning_rate": 2.88814548066504e-06, "loss": 0.1571, "reward": 0.3333333432674408, "reward_std": 0.3872983753681183, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 317 }, { "clip_ratio": 0.0, "completion_length": 716.0833587646484, "epoch": 0.212, "grad_norm": 1.020689606666565, "kl": 0.819091796875, "learning_rate": 2.8868190504184698e-06, "loss": 0.017, "reward": 0.2500000074505806, "reward_std": 0.3872983753681183, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 318 }, { "clip_ratio": 0.0, "completion_length": 650.7916870117188, "epoch": 0.21266666666666667, "grad_norm": 0.6116288900375366, "kl": 0.67529296875, "learning_rate": 2.8854851099665622e-06, "loss": 0.071, "reward": 0.2083333358168602, "reward_std": 0.3410547897219658, "rewards/accuracy_reward": 0.2083333358168602, "rewards/format_reward": 0.0, "step": 319 }, { "clip_ratio": 0.0, "completion_length": 725.6666717529297, "epoch": 0.21333333333333335, "grad_norm": 1.2135902643203735, "kl": 0.9560546875, "learning_rate": 2.8841436665331635e-06, "loss": 0.1567, "reward": 0.3333333469927311, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.3333333469927311, "rewards/format_reward": 0.0, "step": 320 }, { "clip_ratio": 0.0, "completion_length": 612.208366394043, "epoch": 0.214, "grad_norm": 1.4188835620880127, "kl": 2.01348876953125, "learning_rate": 2.8827947273827507e-06, "loss": 0.1081, "reward": 0.4583333358168602, "reward_std": 0.539891391992569, "rewards/accuracy_reward": 0.4583333358168602, "rewards/format_reward": 0.0, "step": 321 }, { "clip_ratio": 0.0, "completion_length": 539.7500228881836, "epoch": 0.21466666666666667, "grad_norm": 1.6991406679153442, "kl": 0.9111328125, "learning_rate": 2.881438299820394e-06, "loss": 0.2498, "reward": 0.6250000223517418, "reward_std": 0.4701542481780052, "rewards/accuracy_reward": 0.6250000223517418, "rewards/format_reward": 0.0, "step": 322 }, { "clip_ratio": 0.0, "completion_length": 685.5416793823242, "epoch": 0.21533333333333332, "grad_norm": 1.8417224884033203, "kl": 2.512939453125, "learning_rate": 2.880074391191716e-06, "loss": 0.1209, "reward": 0.2500000074505806, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 323 }, { "clip_ratio": 0.0, "completion_length": 800.3333587646484, "epoch": 0.216, "grad_norm": 1.7103585004806519, "kl": 2.263671875, "learning_rate": 2.878703008882852e-06, "loss": 0.0929, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 324 }, { "clip_ratio": 0.0, "completion_length": 514.1250190734863, "epoch": 0.21666666666666667, "grad_norm": 0.9776443243026733, "kl": 0.911865234375, "learning_rate": 2.8773241603204113e-06, "loss": 0.1949, "reward": 0.666666679084301, "reward_std": 0.3872983753681183, "rewards/accuracy_reward": 0.666666679084301, "rewards/format_reward": 0.0, "step": 325 }, { "clip_ratio": 0.0, "completion_length": 637.7916717529297, "epoch": 0.21733333333333332, "grad_norm": 0.7969886064529419, "kl": 0.54443359375, "learning_rate": 2.8759378529714358e-06, "loss": 0.1051, "reward": 0.3750000074505806, "reward_std": 0.39512956142425537, "rewards/accuracy_reward": 0.3750000074505806, "rewards/format_reward": 0.0, "step": 326 }, { "clip_ratio": 0.0, "completion_length": 691.0833511352539, "epoch": 0.218, "grad_norm": 0.568488359451294, "kl": 1.050537109375, "learning_rate": 2.8745440943433595e-06, "loss": 0.0732, "reward": 0.3750000037252903, "reward_std": 0.3061862140893936, "rewards/accuracy_reward": 0.3750000037252903, "rewards/format_reward": 0.0, "step": 327 }, { "clip_ratio": 0.0, "completion_length": 601.2500305175781, "epoch": 0.21866666666666668, "grad_norm": 1.9067169427871704, "kl": 1.5670166015625, "learning_rate": 2.8731428919839684e-06, "loss": 0.244, "reward": 0.3750000111758709, "reward_std": 0.4893604516983032, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 328 }, { "clip_ratio": 0.0, "completion_length": 663.6250152587891, "epoch": 0.21933333333333332, "grad_norm": 0.6781679391860962, "kl": 1.529296875, "learning_rate": 2.8717342534813604e-06, "loss": 0.1593, "reward": 0.291666679084301, "reward_std": 0.306186206638813, "rewards/accuracy_reward": 0.291666679084301, "rewards/format_reward": 0.0, "step": 329 }, { "clip_ratio": 0.0, "completion_length": 748.9583587646484, "epoch": 0.22, "grad_norm": 0.8124555349349976, "kl": 1.416015625, "learning_rate": 2.8703181864639013e-06, "loss": 0.1841, "reward": 0.2500000037252903, "reward_std": 0.44311685860157013, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.0, "step": 330 }, { "clip_ratio": 0.0, "completion_length": 666.5000228881836, "epoch": 0.22066666666666668, "grad_norm": 0.7721891403198242, "kl": 1.623046875, "learning_rate": 2.8688946986001866e-06, "loss": 0.1924, "reward": 0.2500000037252903, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.0, "step": 331 }, { "clip_ratio": 0.0, "completion_length": 629.2083587646484, "epoch": 0.22133333333333333, "grad_norm": 1.5475730895996094, "kl": 1.8046875, "learning_rate": 2.867463797598999e-06, "loss": 0.2545, "reward": 0.2916666753590107, "reward_std": 0.3602609932422638, "rewards/accuracy_reward": 0.2916666753590107, "rewards/format_reward": 0.0, "step": 332 }, { "clip_ratio": 0.0, "completion_length": 846.6666870117188, "epoch": 0.222, "grad_norm": 1.1416436433792114, "kl": 2.83984375, "learning_rate": 2.8660254912092654e-06, "loss": 0.1612, "reward": 0.0833333358168602, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 333 }, { "clip_ratio": 0.0, "completion_length": 709.6250152587891, "epoch": 0.22266666666666668, "grad_norm": 0.5469626188278198, "kl": 2.013671875, "learning_rate": 2.8645797872200178e-06, "loss": 0.2266, "reward": 0.2500000074505806, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 334 }, { "clip_ratio": 0.0, "completion_length": 744.1250152587891, "epoch": 0.22333333333333333, "grad_norm": 1.199988603591919, "kl": 1.630859375, "learning_rate": 2.8631266934603466e-06, "loss": 0.0002, "reward": 0.2500000037252903, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.0, "step": 335 }, { "clip_ratio": 0.0, "completion_length": 815.5000152587891, "epoch": 0.224, "grad_norm": 1.045061707496643, "kl": 2.37890625, "learning_rate": 2.861666217799363e-06, "loss": 0.2694, "reward": 0.2500000037252903, "reward_std": 0.3680921792984009, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.0, "step": 336 }, { "clip_ratio": 0.0, "completion_length": 762.0833435058594, "epoch": 0.22466666666666665, "grad_norm": 1.386441946029663, "kl": 1.69842529296875, "learning_rate": 2.860198368146154e-06, "loss": 0.0696, "reward": 0.25, "reward_std": 0.0, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "step": 337 }, { "clip_ratio": 0.0, "completion_length": 786.5000152587891, "epoch": 0.22533333333333333, "grad_norm": 0.620394766330719, "kl": 1.5791015625, "learning_rate": 2.8587231524497397e-06, "loss": 0.1336, "reward": 0.2083333358168602, "reward_std": 0.3410547822713852, "rewards/accuracy_reward": 0.2083333358168602, "rewards/format_reward": 0.0, "step": 338 }, { "clip_ratio": 0.0, "completion_length": 744.5000152587891, "epoch": 0.226, "grad_norm": 1.0943931341171265, "kl": 1.185546875, "learning_rate": 2.8572405786990296e-06, "loss": 0.0881, "reward": 0.291666679084301, "reward_std": 0.306186206638813, "rewards/accuracy_reward": 0.291666679084301, "rewards/format_reward": 0.0, "step": 339 }, { "clip_ratio": 0.0, "completion_length": 815.2083435058594, "epoch": 0.22666666666666666, "grad_norm": 1.2709459066390991, "kl": 1.296875, "learning_rate": 2.855750654922781e-06, "loss": 0.2645, "reward": 0.2916666716337204, "reward_std": 0.4701542407274246, "rewards/accuracy_reward": 0.2916666716337204, "rewards/format_reward": 0.0, "step": 340 }, { "clip_ratio": 0.0, "completion_length": 746.7916717529297, "epoch": 0.22733333333333333, "grad_norm": 1.8994284868240356, "kl": 1.1435546875, "learning_rate": 2.8542533891895543e-06, "loss": 0.1436, "reward": 0.2916666753590107, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.2916666753590107, "rewards/format_reward": 0.0, "step": 341 }, { "clip_ratio": 0.0, "completion_length": 590.9166870117188, "epoch": 0.228, "grad_norm": 1.416043758392334, "kl": 0.62255859375, "learning_rate": 2.852748789607671e-06, "loss": 0.1184, "reward": 0.5000000149011612, "reward_std": 0.5163978338241577, "rewards/accuracy_reward": 0.5000000149011612, "rewards/format_reward": 0.0, "step": 342 }, { "clip_ratio": 0.0, "completion_length": 689.3333587646484, "epoch": 0.22866666666666666, "grad_norm": 0.7572773098945618, "kl": 1.896484375, "learning_rate": 2.8512368643251656e-06, "loss": 0.1324, "reward": 0.2916666716337204, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.2916666716337204, "rewards/format_reward": 0.0, "step": 343 }, { "clip_ratio": 0.0, "completion_length": 735.0000305175781, "epoch": 0.22933333333333333, "grad_norm": 1.9143905639648438, "kl": 2.509765625, "learning_rate": 2.8497176215297474e-06, "loss": 0.3188, "reward": 0.25, "reward_std": 0.273861289024353, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "step": 344 }, { "clip_ratio": 0.0, "completion_length": 723.5416870117188, "epoch": 0.23, "grad_norm": 3.1349780559539795, "kl": 4.560546875, "learning_rate": 2.8481910694487506e-06, "loss": 0.3491, "reward": 0.1666666716337204, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 345 }, { "clip_ratio": 0.0, "completion_length": 508.87501525878906, "epoch": 0.23066666666666666, "grad_norm": 1.69258451461792, "kl": 1.5439453125, "learning_rate": 2.846657216349094e-06, "loss": 0.2047, "reward": 0.4583333507180214, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.4583333507180214, "rewards/format_reward": 0.0, "step": 346 }, { "clip_ratio": 0.0, "completion_length": 642.8333511352539, "epoch": 0.23133333333333334, "grad_norm": 1.8292394876480103, "kl": 2.66015625, "learning_rate": 2.845116070537234e-06, "loss": 0.1926, "reward": 0.2083333395421505, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 347 }, { "clip_ratio": 0.0, "completion_length": 685.3333511352539, "epoch": 0.232, "grad_norm": 2.656902551651001, "kl": 3.1328125, "learning_rate": 2.8435676403591196e-06, "loss": 0.2938, "reward": 0.2083333395421505, "reward_std": 0.3602609932422638, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 348 }, { "clip_ratio": 0.0, "completion_length": 587.7083587646484, "epoch": 0.23266666666666666, "grad_norm": 2.233133316040039, "kl": 2.2265625, "learning_rate": 2.8420119342001475e-06, "loss": 0.2647, "reward": 0.2916666716337204, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.2916666716337204, "rewards/format_reward": 0.0, "step": 349 }, { "clip_ratio": 0.0, "completion_length": 725.2500305175781, "epoch": 0.23333333333333334, "grad_norm": 2.177471399307251, "kl": 3.642578125, "learning_rate": 2.8404489604851183e-06, "loss": 0.2565, "reward": 0.1666666679084301, "reward_std": 0.23899272084236145, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 350 }, { "clip_ratio": 0.0, "completion_length": 671.0000228881836, "epoch": 0.234, "grad_norm": 0.6246495246887207, "kl": 1.1259765625, "learning_rate": 2.8388787276781884e-06, "loss": 0.0978, "reward": 0.291666679084301, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.291666679084301, "rewards/format_reward": 0.0, "step": 351 }, { "clip_ratio": 0.0, "completion_length": 695.6250152587891, "epoch": 0.23466666666666666, "grad_norm": 6.828301906585693, "kl": 3.259765625, "learning_rate": 2.837301244282825e-06, "loss": 0.4126, "reward": 0.2500000074505806, "reward_std": 0.3872983753681183, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 352 }, { "clip_ratio": 0.0, "completion_length": 637.3750305175781, "epoch": 0.23533333333333334, "grad_norm": 0.5631955862045288, "kl": 1.646484375, "learning_rate": 2.8357165188417604e-06, "loss": 0.1263, "reward": 0.3750000074505806, "reward_std": 0.4701542481780052, "rewards/accuracy_reward": 0.3750000074505806, "rewards/format_reward": 0.0, "step": 353 }, { "clip_ratio": 0.0, "completion_length": 688.2083587646484, "epoch": 0.236, "grad_norm": 0.6810453534126282, "kl": 1.085693359375, "learning_rate": 2.8341245599369467e-06, "loss": 0.1206, "reward": 0.2083333395421505, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 354 }, { "clip_ratio": 0.0, "completion_length": 464.9583511352539, "epoch": 0.23666666666666666, "grad_norm": 1.8797169923782349, "kl": 1.536865234375, "learning_rate": 2.8325253761895064e-06, "loss": 0.2456, "reward": 0.3750000111758709, "reward_std": 0.4893604442477226, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 355 }, { "clip_ratio": 0.0, "completion_length": 712.5416946411133, "epoch": 0.23733333333333334, "grad_norm": 1.2633169889450073, "kl": 1.607421875, "learning_rate": 2.830918976259689e-06, "loss": 0.1627, "reward": 0.2500000111758709, "reward_std": 0.20412415266036987, "rewards/accuracy_reward": 0.2500000111758709, "rewards/format_reward": 0.0, "step": 356 }, { "clip_ratio": 0.0, "completion_length": 590.1666717529297, "epoch": 0.238, "grad_norm": 2.0736844539642334, "kl": 2.0, "learning_rate": 2.829305368846822e-06, "loss": 0.2444, "reward": 0.416666679084301, "reward_std": 0.44311685860157013, "rewards/accuracy_reward": 0.416666679084301, "rewards/format_reward": 0.0, "step": 357 }, { "clip_ratio": 0.0, "completion_length": 620.1666793823242, "epoch": 0.23866666666666667, "grad_norm": 1.328733205795288, "kl": 2.529296875, "learning_rate": 2.827684562689265e-06, "loss": 0.2983, "reward": 0.3333333432674408, "reward_std": 0.3872983753681183, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 358 }, { "clip_ratio": 0.0, "completion_length": 788.6250152587891, "epoch": 0.23933333333333334, "grad_norm": 3.97040057182312, "kl": 3.16796875, "learning_rate": 2.8260565665643612e-06, "loss": 0.1523, "reward": 0.1666666679084301, "reward_std": 0.23899272084236145, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 359 }, { "clip_ratio": 0.0, "completion_length": 722.0833435058594, "epoch": 0.24, "grad_norm": 3.513559103012085, "kl": 3.4794921875, "learning_rate": 2.8244213892883906e-06, "loss": 0.0847, "reward": 0.2083333358168602, "reward_std": 0.3410547897219658, "rewards/accuracy_reward": 0.2083333358168602, "rewards/format_reward": 0.0, "step": 360 }, { "clip_ratio": 0.0, "completion_length": 764.3750305175781, "epoch": 0.24066666666666667, "grad_norm": 1.3380812406539917, "kl": 2.2578125, "learning_rate": 2.822779039716522e-06, "loss": 0.087, "reward": 0.1666666716337204, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 361 }, { "clip_ratio": 0.0, "completion_length": 432.0833396911621, "epoch": 0.24133333333333334, "grad_norm": 1.8138567209243774, "kl": 2.32476806640625, "learning_rate": 2.821129526742766e-06, "loss": 0.1629, "reward": 0.4583333432674408, "reward_std": 0.39512956142425537, "rewards/accuracy_reward": 0.4583333432674408, "rewards/format_reward": 0.0, "step": 362 }, { "clip_ratio": 0.0, "completion_length": 639.6666870117188, "epoch": 0.242, "grad_norm": 2.672344207763672, "kl": 0.936737060546875, "learning_rate": 2.819472859299925e-06, "loss": 0.1909, "reward": 0.3750000111758709, "reward_std": 0.3410547897219658, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 363 }, { "clip_ratio": 0.0, "completion_length": 848.6666870117188, "epoch": 0.24266666666666667, "grad_norm": 0.5916891098022461, "kl": 0.815521240234375, "learning_rate": 2.8178090463595464e-06, "loss": 0.0802, "reward": 0.2500000037252903, "reward_std": 0.44311685860157013, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.0, "step": 364 }, { "clip_ratio": 0.0, "completion_length": 835.2083587646484, "epoch": 0.24333333333333335, "grad_norm": 0.42434000968933105, "kl": 1.2255859375, "learning_rate": 2.8161380969318728e-06, "loss": 0.1056, "reward": 0.1666666716337204, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 365 }, { "clip_ratio": 0.0, "completion_length": 754.6666870117188, "epoch": 0.244, "grad_norm": 0.4403095245361328, "kl": 0.58984375, "learning_rate": 2.814460020065795e-06, "loss": 0.0919, "reward": 0.1666666679084301, "reward_std": 0.23899272084236145, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 366 }, { "clip_ratio": 0.0, "completion_length": 570.2083396911621, "epoch": 0.24466666666666667, "grad_norm": 2.2124242782592773, "kl": 1.208984375, "learning_rate": 2.812774824848802e-06, "loss": 0.0938, "reward": 0.2500000074505806, "reward_std": 0.3872983753681183, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 367 }, { "clip_ratio": 0.0, "completion_length": 580.875, "epoch": 0.24533333333333332, "grad_norm": 1.664440631866455, "kl": 0.544647216796875, "learning_rate": 2.8110825204069292e-06, "loss": 0.1194, "reward": 0.3750000111758709, "reward_std": 0.3410547822713852, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 368 }, { "clip_ratio": 0.0, "completion_length": 632.2916870117188, "epoch": 0.246, "grad_norm": 1.1837868690490723, "kl": 2.09375, "learning_rate": 2.8093831159047146e-06, "loss": 0.2145, "reward": 0.2083333395421505, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 369 }, { "clip_ratio": 0.0, "completion_length": 813.5417022705078, "epoch": 0.24666666666666667, "grad_norm": 0.7740781903266907, "kl": 1.0126953125, "learning_rate": 2.8076766205451433e-06, "loss": 0.1234, "reward": 0.2500000074505806, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 370 }, { "clip_ratio": 0.0, "completion_length": 790.4167022705078, "epoch": 0.24733333333333332, "grad_norm": 0.5674356818199158, "kl": 0.6036376953125, "learning_rate": 2.8059630435696028e-06, "loss": 0.1305, "reward": 0.2916666716337204, "reward_std": 0.39512956142425537, "rewards/accuracy_reward": 0.2916666716337204, "rewards/format_reward": 0.0, "step": 371 }, { "clip_ratio": 0.0, "completion_length": 799.7083587646484, "epoch": 0.248, "grad_norm": 2.0081496238708496, "kl": 1.453125, "learning_rate": 2.8042423942578284e-06, "loss": 0.2232, "reward": 0.2083333358168602, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.2083333358168602, "rewards/format_reward": 0.0, "step": 372 }, { "clip_ratio": 0.0, "completion_length": 694.5833587646484, "epoch": 0.24866666666666667, "grad_norm": 1.747820258140564, "kl": 2.2392578125, "learning_rate": 2.8025146819278565e-06, "loss": 0.2008, "reward": 0.2083333395421505, "reward_std": 0.3602609932422638, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 373 }, { "clip_ratio": 0.0, "completion_length": 701.2916870117188, "epoch": 0.24933333333333332, "grad_norm": 1.169891595840454, "kl": 1.916015625, "learning_rate": 2.800779915935972e-06, "loss": 0.1971, "reward": 0.3333333395421505, "reward_std": 0.3680921792984009, "rewards/accuracy_reward": 0.3333333395421505, "rewards/format_reward": 0.0, "step": 374 }, { "clip_ratio": 0.0, "completion_length": 733.6666870117188, "epoch": 0.25, "grad_norm": 0.9732698202133179, "kl": 2.212890625, "learning_rate": 2.7990381056766585e-06, "loss": 0.2239, "reward": 0.2500000037252903, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.0, "step": 375 }, { "clip_ratio": 0.0, "completion_length": 648.5833435058594, "epoch": 0.25066666666666665, "grad_norm": 0.8586495518684387, "kl": 1.244140625, "learning_rate": 2.7972892605825464e-06, "loss": 0.1309, "reward": 0.2500000074505806, "reward_std": 0.3332236111164093, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 376 }, { "clip_ratio": 0.0, "completion_length": 779.7500152587891, "epoch": 0.25133333333333335, "grad_norm": 1.4066340923309326, "kl": 1.27001953125, "learning_rate": 2.7955333901243647e-06, "loss": 0.088, "reward": 0.2083333395421505, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 377 }, { "clip_ratio": 0.0, "completion_length": 544.4583511352539, "epoch": 0.252, "grad_norm": 1.1236261129379272, "kl": 1.658203125, "learning_rate": 2.7937705038108863e-06, "loss": 0.3432, "reward": 0.3750000111758709, "reward_std": 0.3602609932422638, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 378 }, { "clip_ratio": 0.0, "completion_length": 606.3333587646484, "epoch": 0.25266666666666665, "grad_norm": 1.1282670497894287, "kl": 1.044921875, "learning_rate": 2.7920006111888777e-06, "loss": 0.1552, "reward": 0.3750000149011612, "reward_std": 0.43528567999601364, "rewards/accuracy_reward": 0.3750000149011612, "rewards/format_reward": 0.0, "step": 379 }, { "clip_ratio": 0.0, "completion_length": 728.5416870117188, "epoch": 0.25333333333333335, "grad_norm": 1.089224100112915, "kl": 1.1103515625, "learning_rate": 2.7902237218430485e-06, "loss": 0.0684, "reward": 0.1666666716337204, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 380 }, { "clip_ratio": 0.0, "completion_length": 757.3333587646484, "epoch": 0.254, "grad_norm": 0.7247681021690369, "kl": 1.2548828125, "learning_rate": 2.7884398453959967e-06, "loss": 0.1629, "reward": 0.2083333395421505, "reward_std": 0.43528567254543304, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 381 }, { "clip_ratio": 0.0, "completion_length": 546.0833358764648, "epoch": 0.25466666666666665, "grad_norm": 0.7020712494850159, "kl": 0.85986328125, "learning_rate": 2.7866489915081606e-06, "loss": 0.037, "reward": 0.5000000037252903, "reward_std": 0.3680921792984009, "rewards/accuracy_reward": 0.5000000037252903, "rewards/format_reward": 0.0, "step": 382 }, { "clip_ratio": 0.0, "completion_length": 625.0000228881836, "epoch": 0.25533333333333336, "grad_norm": 1.6586428880691528, "kl": 0.796630859375, "learning_rate": 2.784851169877763e-06, "loss": 0.1474, "reward": 0.2916666753590107, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.2916666753590107, "rewards/format_reward": 0.0, "step": 383 }, { "clip_ratio": 0.0, "completion_length": 680.0000305175781, "epoch": 0.256, "grad_norm": 0.9505209922790527, "kl": 0.944091796875, "learning_rate": 2.78304639024076e-06, "loss": 0.1374, "reward": 0.5000000074505806, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.5000000074505806, "rewards/format_reward": 0.0, "step": 384 }, { "clip_ratio": 0.0, "completion_length": 648.3750305175781, "epoch": 0.25666666666666665, "grad_norm": 0.4205808639526367, "kl": 1.1171875, "learning_rate": 2.7812346623707886e-06, "loss": 0.1796, "reward": 0.1666666716337204, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 385 }, { "clip_ratio": 0.0, "completion_length": 715.0833587646484, "epoch": 0.25733333333333336, "grad_norm": 1.1350734233856201, "kl": 1.2900390625, "learning_rate": 2.7794159960791125e-06, "loss": 0.1434, "reward": 0.2916666716337204, "reward_std": 0.39512956142425537, "rewards/accuracy_reward": 0.2916666716337204, "rewards/format_reward": 0.0, "step": 386 }, { "clip_ratio": 0.0, "completion_length": 685.0416793823242, "epoch": 0.258, "grad_norm": 2.6464123725891113, "kl": 1.3046875, "learning_rate": 2.7775904012145713e-06, "loss": 0.2836, "reward": 0.4583333432674408, "reward_std": 0.37592335790395737, "rewards/accuracy_reward": 0.4583333432674408, "rewards/format_reward": 0.0, "step": 387 }, { "clip_ratio": 0.0, "completion_length": 499.08333587646484, "epoch": 0.25866666666666666, "grad_norm": 2.5553035736083984, "kl": 1.16552734375, "learning_rate": 2.775757887663525e-06, "loss": 0.0739, "reward": 0.3333333395421505, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.3333333395421505, "rewards/format_reward": 0.0, "step": 388 }, { "clip_ratio": 0.0, "completion_length": 830.0000305175781, "epoch": 0.25933333333333336, "grad_norm": 8.101317405700684, "kl": 3.929595947265625, "learning_rate": 2.7739184653498002e-06, "loss": 0.1722, "reward": 0.2083333395421505, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 389 }, { "clip_ratio": 0.0, "completion_length": 591.4583396911621, "epoch": 0.26, "grad_norm": 2.4231505393981934, "kl": 1.904296875, "learning_rate": 2.772072144234639e-06, "loss": 0.057, "reward": 0.3333333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 390 }, { "clip_ratio": 0.0, "completion_length": 591.4583587646484, "epoch": 0.26066666666666666, "grad_norm": 0.2766169607639313, "kl": 0.64080810546875, "learning_rate": 2.7702189343166426e-06, "loss": 0.1016, "reward": 0.1666666679084301, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 391 }, { "clip_ratio": 0.0, "completion_length": 594.0833587646484, "epoch": 0.2613333333333333, "grad_norm": 1.0892994403839111, "kl": 0.62310791015625, "learning_rate": 2.7683588456317177e-06, "loss": 0.0226, "reward": 0.1666666716337204, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 392 }, { "clip_ratio": 0.0, "completion_length": 419.7083435058594, "epoch": 0.262, "grad_norm": 0.5199337601661682, "kl": 0.534423828125, "learning_rate": 2.7664918882530226e-06, "loss": 0.1227, "reward": 0.6250000149011612, "reward_std": 0.5242290198802948, "rewards/accuracy_reward": 0.6250000149011612, "rewards/format_reward": 0.0, "step": 393 }, { "clip_ratio": 0.0, "completion_length": 544.2917022705078, "epoch": 0.26266666666666666, "grad_norm": 0.31711310148239136, "kl": 0.6705322265625, "learning_rate": 2.764618072290913e-06, "loss": 0.051, "reward": 0.0833333358168602, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 394 }, { "clip_ratio": 0.0, "completion_length": 530.3750152587891, "epoch": 0.2633333333333333, "grad_norm": 0.6411799788475037, "kl": 1.166015625, "learning_rate": 2.7627374078928863e-06, "loss": 0.0446, "reward": 0.1666666716337204, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 395 }, { "clip_ratio": 0.0, "completion_length": 634.6666870117188, "epoch": 0.264, "grad_norm": 2.982640266418457, "kl": 1.0498046875, "learning_rate": 2.7608499052435266e-06, "loss": 0.2338, "reward": 0.2500000037252903, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.0, "step": 396 }, { "clip_ratio": 0.0, "completion_length": 575.3750305175781, "epoch": 0.26466666666666666, "grad_norm": 0.5314953327178955, "kl": 0.628173828125, "learning_rate": 2.7589555745644502e-06, "loss": 0.0425, "reward": 0.125, "reward_std": 0.1369306445121765, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "step": 397 }, { "clip_ratio": 0.0, "completion_length": 274.54167556762695, "epoch": 0.2653333333333333, "grad_norm": 0.847283661365509, "kl": 0.2261962890625, "learning_rate": 2.757054426114251e-06, "loss": 0.043, "reward": 0.5833333432674408, "reward_std": 0.5320602059364319, "rewards/accuracy_reward": 0.5833333432674408, "rewards/format_reward": 0.0, "step": 398 }, { "clip_ratio": 0.0, "completion_length": 360.0416793823242, "epoch": 0.266, "grad_norm": 0.32528355717658997, "kl": 0.14752197265625, "learning_rate": 2.7551464701884427e-06, "loss": 0.1017, "reward": 0.6250000298023224, "reward_std": 0.306186206638813, "rewards/accuracy_reward": 0.6250000298023224, "rewards/format_reward": 0.0, "step": 399 }, { "clip_ratio": 0.0, "completion_length": 586.2083587646484, "epoch": 0.26666666666666666, "grad_norm": 0.7450648546218872, "kl": 0.89208984375, "learning_rate": 2.753231717119405e-06, "loss": 0.1473, "reward": 0.375, "reward_std": 0.41079193353652954, "rewards/accuracy_reward": 0.375, "rewards/format_reward": 0.0, "step": 400 }, { "clip_ratio": 0.0, "completion_length": 553.2916870117188, "epoch": 0.2673333333333333, "grad_norm": 0.2396654635667801, "kl": 0.402587890625, "learning_rate": 2.751310177276326e-06, "loss": 0.0082, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 401 }, { "clip_ratio": 0.0, "completion_length": 673.7916870117188, "epoch": 0.268, "grad_norm": 0.35151779651641846, "kl": 1.1044921875, "learning_rate": 2.749381861065149e-06, "loss": 0.1182, "reward": 0.1666666679084301, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 402 }, { "clip_ratio": 0.0, "completion_length": 621.3750152587891, "epoch": 0.26866666666666666, "grad_norm": 0.6206917762756348, "kl": 1.134765625, "learning_rate": 2.7474467789285127e-06, "loss": 0.0481, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 403 }, { "clip_ratio": 0.0, "completion_length": 654.4583587646484, "epoch": 0.2693333333333333, "grad_norm": 0.26414692401885986, "kl": 0.79296875, "learning_rate": 2.7455049413456964e-06, "loss": 0.0925, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 404 }, { "clip_ratio": 0.0, "completion_length": 505.20835876464844, "epoch": 0.27, "grad_norm": 0.5914087891578674, "kl": 0.73828125, "learning_rate": 2.7435563588325624e-06, "loss": 0.0638, "reward": 0.2916666753590107, "reward_std": 0.3602609932422638, "rewards/accuracy_reward": 0.2916666753590107, "rewards/format_reward": 0.0, "step": 405 }, { "clip_ratio": 0.0, "completion_length": 591.6250305175781, "epoch": 0.27066666666666667, "grad_norm": 0.7394228577613831, "kl": 1.1201171875, "learning_rate": 2.741601041941501e-06, "loss": 0.1307, "reward": 0.2083333395421505, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 406 }, { "clip_ratio": 0.0, "completion_length": 591.5416870117188, "epoch": 0.2713333333333333, "grad_norm": 0.34116968512535095, "kl": 0.888671875, "learning_rate": 2.7396390012613706e-06, "loss": -0.0416, "reward": 0.0833333358168602, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 407 }, { "clip_ratio": 0.0, "completion_length": 679.8750228881836, "epoch": 0.272, "grad_norm": 0.3498900234699249, "kl": 1.125, "learning_rate": 2.7376702474174426e-06, "loss": 0.1093, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 408 }, { "clip_ratio": 0.0, "completion_length": 787.2083587646484, "epoch": 0.27266666666666667, "grad_norm": 0.404981404542923, "kl": 0.52490234375, "learning_rate": 2.735694791071343e-06, "loss": 0.0792, "reward": 0.1250000037252903, "reward_std": 0.3061862140893936, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 409 }, { "clip_ratio": 0.0, "completion_length": 543.6666793823242, "epoch": 0.2733333333333333, "grad_norm": 0.24135825037956238, "kl": 0.39501953125, "learning_rate": 2.7337126429209934e-06, "loss": 0.0695, "reward": 0.2083333358168602, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.2083333358168602, "rewards/format_reward": 0.0, "step": 410 }, { "clip_ratio": 0.0, "completion_length": 628.7500305175781, "epoch": 0.274, "grad_norm": 0.41406717896461487, "kl": 0.355224609375, "learning_rate": 2.7317238137005563e-06, "loss": 0.051, "reward": 0.2083333395421505, "reward_std": 0.43528566509485245, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 411 }, { "clip_ratio": 0.0, "completion_length": 673.7916870117188, "epoch": 0.27466666666666667, "grad_norm": 0.5244023203849792, "kl": 0.417236328125, "learning_rate": 2.729728314180373e-06, "loss": 0.0622, "reward": 0.2500000074505806, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 412 }, { "clip_ratio": 0.0, "completion_length": 560.3750076293945, "epoch": 0.2753333333333333, "grad_norm": 0.7630873918533325, "kl": 0.32080078125, "learning_rate": 2.727726155166909e-06, "loss": 0.028, "reward": 0.4583333432674408, "reward_std": 0.39512956142425537, "rewards/accuracy_reward": 0.4583333432674408, "rewards/format_reward": 0.0, "step": 413 }, { "clip_ratio": 0.0, "completion_length": 638.3333435058594, "epoch": 0.276, "grad_norm": 0.2540242075920105, "kl": 0.270263671875, "learning_rate": 2.725717347502693e-06, "loss": 0.0566, "reward": 0.1666666716337204, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 414 }, { "clip_ratio": 0.0, "completion_length": 553.5416870117188, "epoch": 0.27666666666666667, "grad_norm": 0.5088773369789124, "kl": 0.28009033203125, "learning_rate": 2.7237019020662585e-06, "loss": 0.1033, "reward": 0.5000000186264515, "reward_std": 0.46232306957244873, "rewards/accuracy_reward": 0.5000000186264515, "rewards/format_reward": 0.0, "step": 415 }, { "clip_ratio": 0.0, "completion_length": 475.3333435058594, "epoch": 0.2773333333333333, "grad_norm": 0.9551652073860168, "kl": 0.419189453125, "learning_rate": 2.7216798297720855e-06, "loss": 0.0799, "reward": 0.4166666828095913, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.4166666828095913, "rewards/format_reward": 0.0, "step": 416 }, { "clip_ratio": 0.0, "completion_length": 636.7083587646484, "epoch": 0.278, "grad_norm": 0.6511146426200867, "kl": 0.252685546875, "learning_rate": 2.7196511415705416e-06, "loss": 0.1677, "reward": 0.3333333432674408, "reward_std": 0.3872983753681183, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 417 }, { "clip_ratio": 0.0, "completion_length": 668.6666717529297, "epoch": 0.2786666666666667, "grad_norm": 0.41466885805130005, "kl": 0.7294921875, "learning_rate": 2.7176158484478224e-06, "loss": 0.0545, "reward": 0.2500000074505806, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 418 }, { "clip_ratio": 0.0, "completion_length": 574.7083511352539, "epoch": 0.2793333333333333, "grad_norm": 0.3017953932285309, "kl": 0.47119140625, "learning_rate": 2.7155739614258917e-06, "loss": 0.0925, "reward": 0.1666666716337204, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 419 }, { "clip_ratio": 0.0, "completion_length": 571.625, "epoch": 0.28, "grad_norm": 0.2564389109611511, "kl": 0.194183349609375, "learning_rate": 2.713525491562421e-06, "loss": 0.0601, "reward": 0.5833333395421505, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.5833333395421505, "rewards/format_reward": 0.0, "step": 420 }, { "clip_ratio": 0.0, "completion_length": 593.8333358764648, "epoch": 0.2806666666666667, "grad_norm": 0.8355961441993713, "kl": 1.33642578125, "learning_rate": 2.711470449950733e-06, "loss": 0.144, "reward": 0.2500000111758709, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.2500000111758709, "rewards/format_reward": 0.0, "step": 421 }, { "clip_ratio": 0.0, "completion_length": 727.8750305175781, "epoch": 0.2813333333333333, "grad_norm": 0.41848504543304443, "kl": 0.9892578125, "learning_rate": 2.709408847719737e-06, "loss": 0.1842, "reward": 0.291666679084301, "reward_std": 0.3061862215399742, "rewards/accuracy_reward": 0.291666679084301, "rewards/format_reward": 0.0, "step": 422 }, { "clip_ratio": 0.0, "completion_length": 687.5416870117188, "epoch": 0.282, "grad_norm": 0.41793835163116455, "kl": 0.55029296875, "learning_rate": 2.7073406960338713e-06, "loss": 0.0513, "reward": 0.3333333358168602, "reward_std": 0.40296074748039246, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 423 }, { "clip_ratio": 0.0, "completion_length": 571.4166870117188, "epoch": 0.2826666666666667, "grad_norm": 0.388156920671463, "kl": 0.507568359375, "learning_rate": 2.705266006093043e-06, "loss": 0.0457, "reward": 0.291666679084301, "reward_std": 0.3061862289905548, "rewards/accuracy_reward": 0.291666679084301, "rewards/format_reward": 0.0, "step": 424 }, { "clip_ratio": 0.0, "completion_length": 619.5000114440918, "epoch": 0.2833333333333333, "grad_norm": 0.5542228817939758, "kl": 0.48291015625, "learning_rate": 2.7031847891325657e-06, "loss": 0.0919, "reward": 0.4166666828095913, "reward_std": 0.46232306212186813, "rewards/accuracy_reward": 0.4166666828095913, "rewards/format_reward": 0.0, "step": 425 }, { "clip_ratio": 0.0, "completion_length": 617.7500228881836, "epoch": 0.284, "grad_norm": 1.7153860330581665, "kl": 0.75927734375, "learning_rate": 2.7010970564231e-06, "loss": 0.1626, "reward": 0.3750000074505806, "reward_std": 0.4701542556285858, "rewards/accuracy_reward": 0.3750000074505806, "rewards/format_reward": 0.0, "step": 426 }, { "clip_ratio": 0.0, "completion_length": 655.0833587646484, "epoch": 0.2846666666666667, "grad_norm": 1.305392861366272, "kl": 0.5380859375, "learning_rate": 2.6990028192705924e-06, "loss": 0.1315, "reward": 0.3333333469927311, "reward_std": 0.40824829041957855, "rewards/accuracy_reward": 0.3333333469927311, "rewards/format_reward": 0.0, "step": 427 }, { "clip_ratio": 0.0, "completion_length": 548.7500228881836, "epoch": 0.2853333333333333, "grad_norm": 1.8265390396118164, "kl": 1.588623046875, "learning_rate": 2.696902089016213e-06, "loss": 0.1727, "reward": 0.5000000074505806, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.5000000074505806, "rewards/format_reward": 0.0, "step": 428 }, { "clip_ratio": 0.0, "completion_length": 464.6666793823242, "epoch": 0.286, "grad_norm": 1.1838024854660034, "kl": 1.4493408203125, "learning_rate": 2.6947948770362945e-06, "loss": 0.2009, "reward": 0.6250000149011612, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.6250000149011612, "rewards/format_reward": 0.0, "step": 429 }, { "clip_ratio": 0.0, "completion_length": 614.6250152587891, "epoch": 0.2866666666666667, "grad_norm": 0.72001051902771, "kl": 1.88916015625, "learning_rate": 2.6926811947422717e-06, "loss": 0.2087, "reward": 0.2500000074505806, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 430 }, { "clip_ratio": 0.0, "completion_length": 480.6666793823242, "epoch": 0.28733333333333333, "grad_norm": 1.211071252822876, "kl": 1.5098876953125, "learning_rate": 2.690561053580619e-06, "loss": 0.0428, "reward": 0.5833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.5833333358168602, "rewards/format_reward": 0.0, "step": 431 }, { "clip_ratio": 0.0, "completion_length": 620.9166870117188, "epoch": 0.288, "grad_norm": 2.368042230606079, "kl": 2.029296875, "learning_rate": 2.688434465032786e-06, "loss": 0.1169, "reward": 0.1250000037252903, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 432 }, { "clip_ratio": 0.0, "completion_length": 629.2083358764648, "epoch": 0.2886666666666667, "grad_norm": 1.3651472330093384, "kl": 1.58978271484375, "learning_rate": 2.6863014406151413e-06, "loss": 0.14, "reward": 0.5000000074505806, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.5000000074505806, "rewards/format_reward": 0.0, "step": 433 }, { "clip_ratio": 0.0, "completion_length": 446.25000762939453, "epoch": 0.28933333333333333, "grad_norm": 0.52852863073349, "kl": 0.58514404296875, "learning_rate": 2.6841619918789038e-06, "loss": -0.047, "reward": 0.4583333469927311, "reward_std": 0.4893604442477226, "rewards/accuracy_reward": 0.4583333469927311, "rewards/format_reward": 0.0, "step": 434 }, { "clip_ratio": 0.0, "completion_length": 687.1250152587891, "epoch": 0.29, "grad_norm": 0.4062546491622925, "kl": 0.9871826171875, "learning_rate": 2.6820161304100827e-06, "loss": 0.0608, "reward": 0.1666666716337204, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 435 }, { "clip_ratio": 0.0, "completion_length": 664.8750152587891, "epoch": 0.2906666666666667, "grad_norm": 0.3365577161312103, "kl": 0.296142578125, "learning_rate": 2.679863867829417e-06, "loss": -0.0031, "reward": 0.2083333395421505, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 436 }, { "clip_ratio": 0.0, "completion_length": 558.3750076293945, "epoch": 0.29133333333333333, "grad_norm": 0.7326304316520691, "kl": 0.582763671875, "learning_rate": 2.677705215792308e-06, "loss": -0.012, "reward": 0.4166666828095913, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.4166666828095913, "rewards/format_reward": 0.0, "step": 437 }, { "clip_ratio": 0.0, "completion_length": 748.0833587646484, "epoch": 0.292, "grad_norm": 0.3996496796607971, "kl": 0.6966552734375, "learning_rate": 2.67554018598876e-06, "loss": 0.0911, "reward": 0.2500000074505806, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 438 }, { "clip_ratio": 0.0, "completion_length": 491.16668701171875, "epoch": 0.2926666666666667, "grad_norm": 0.27381661534309387, "kl": 0.192138671875, "learning_rate": 2.6733687901433143e-06, "loss": 0.035, "reward": 0.2916666753590107, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.2916666753590107, "rewards/format_reward": 0.0, "step": 439 }, { "clip_ratio": 0.0, "completion_length": 552.3750152587891, "epoch": 0.29333333333333333, "grad_norm": 0.5243602991104126, "kl": 0.40716552734375, "learning_rate": 2.671191040014989e-06, "loss": 0.13, "reward": 0.6250000149011612, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.6250000149011612, "rewards/format_reward": 0.0, "step": 440 }, { "clip_ratio": 0.0, "completion_length": 520.8333358764648, "epoch": 0.294, "grad_norm": 1.372970461845398, "kl": 0.37255859375, "learning_rate": 2.6690069473972115e-06, "loss": 0.0853, "reward": 0.3750000111758709, "reward_std": 0.3410547822713852, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 441 }, { "clip_ratio": 0.0, "completion_length": 414.6666793823242, "epoch": 0.2946666666666667, "grad_norm": 0.5989941358566284, "kl": 0.438232421875, "learning_rate": 2.666816524117757e-06, "loss": 0.0623, "reward": 0.7083333432674408, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.7083333432674408, "rewards/format_reward": 0.0, "step": 442 }, { "clip_ratio": 0.0, "completion_length": 696.2500228881836, "epoch": 0.29533333333333334, "grad_norm": 1.2816026210784912, "kl": 1.11962890625, "learning_rate": 2.6646197820386834e-06, "loss": 0.2618, "reward": 0.4583333432674408, "reward_std": 0.37592336535453796, "rewards/accuracy_reward": 0.4583333432674408, "rewards/format_reward": 0.0, "step": 443 }, { "clip_ratio": 0.0, "completion_length": 398.00000762939453, "epoch": 0.296, "grad_norm": 3.200883388519287, "kl": 0.78955078125, "learning_rate": 2.6624167330562694e-06, "loss": 0.2994, "reward": 0.708333358168602, "reward_std": 0.4701542407274246, "rewards/accuracy_reward": 0.708333358168602, "rewards/format_reward": 0.0, "step": 444 }, { "clip_ratio": 0.0, "completion_length": 559.4166946411133, "epoch": 0.2966666666666667, "grad_norm": 1.1091234683990479, "kl": 1.4296875, "learning_rate": 2.660207389100946e-06, "loss": 0.2155, "reward": 0.5000000186264515, "reward_std": 0.46232306212186813, "rewards/accuracy_reward": 0.5000000186264515, "rewards/format_reward": 0.0, "step": 445 }, { "clip_ratio": 0.0, "completion_length": 595.0000152587891, "epoch": 0.29733333333333334, "grad_norm": 3.602403163909912, "kl": 3.1171875, "learning_rate": 2.657991762137235e-06, "loss": 0.2795, "reward": 0.2500000074505806, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 446 }, { "clip_ratio": 0.0, "completion_length": 750.0, "epoch": 0.298, "grad_norm": 2.639753818511963, "kl": 4.03515625, "learning_rate": 2.655769864163684e-06, "loss": 0.341, "reward": 0.291666679084301, "reward_std": 0.3061862140893936, "rewards/accuracy_reward": 0.291666679084301, "rewards/format_reward": 0.0, "step": 447 }, { "clip_ratio": 0.0, "completion_length": 767.4166870117188, "epoch": 0.2986666666666667, "grad_norm": 1.7193154096603394, "kl": 3.2421875, "learning_rate": 2.653541707212799e-06, "loss": 0.1922, "reward": 0.125, "reward_std": 0.1369306445121765, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "step": 448 }, { "clip_ratio": 0.0, "completion_length": 590.6250228881836, "epoch": 0.29933333333333334, "grad_norm": 1.2707158327102661, "kl": 2.9541015625, "learning_rate": 2.651307303350983e-06, "loss": 0.305, "reward": 0.2916666716337204, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.2916666716337204, "rewards/format_reward": 0.0, "step": 449 }, { "clip_ratio": 0.0, "completion_length": 714.8333435058594, "epoch": 0.3, "grad_norm": 1.2895716428756714, "kl": 1.9921875, "learning_rate": 2.649066664678467e-06, "loss": 0.1382, "reward": 0.2083333395421505, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 450 }, { "clip_ratio": 0.0, "completion_length": 798.7916870117188, "epoch": 0.3006666666666667, "grad_norm": 0.9849034547805786, "kl": 1.806640625, "learning_rate": 2.6468198033292474e-06, "loss": 0.1402, "reward": 0.2083333358168602, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.2083333358168602, "rewards/format_reward": 0.0, "step": 451 }, { "clip_ratio": 0.0, "completion_length": 714.7083511352539, "epoch": 0.30133333333333334, "grad_norm": 1.2150938510894775, "kl": 0.8994140625, "learning_rate": 2.6445667314710174e-06, "loss": 0.1323, "reward": 0.3333333469927311, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.3333333469927311, "rewards/format_reward": 0.0, "step": 452 }, { "clip_ratio": 0.0, "completion_length": 605.4583435058594, "epoch": 0.302, "grad_norm": 0.3450285494327545, "kl": 0.9036865234375, "learning_rate": 2.6423074613051055e-06, "loss": 0.113, "reward": 0.291666679084301, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.291666679084301, "rewards/format_reward": 0.0, "step": 453 }, { "clip_ratio": 0.0, "completion_length": 776.9166870117188, "epoch": 0.30266666666666664, "grad_norm": 0.3091043531894684, "kl": 0.90625, "learning_rate": 2.6400420050664027e-06, "loss": 0.0883, "reward": 0.1250000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 454 }, { "clip_ratio": 0.0, "completion_length": 817.5000152587891, "epoch": 0.30333333333333334, "grad_norm": 0.47548991441726685, "kl": 0.42333984375, "learning_rate": 2.6377703750233033e-06, "loss": 0.1359, "reward": 0.2500000037252903, "reward_std": 0.4431168660521507, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.0, "step": 455 }, { "clip_ratio": 0.0, "completion_length": 556.8750228881836, "epoch": 0.304, "grad_norm": 0.15603232383728027, "kl": 0.621673583984375, "learning_rate": 2.6354925834776346e-06, "loss": 0.0916, "reward": 0.5416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.5416666679084301, "rewards/format_reward": 0.0, "step": 456 }, { "clip_ratio": 0.0, "completion_length": 524.2500038146973, "epoch": 0.30466666666666664, "grad_norm": 0.3357190191745758, "kl": 0.48187255859375, "learning_rate": 2.6332086427645907e-06, "loss": 0.069, "reward": 0.2500000037252903, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.0, "step": 457 }, { "clip_ratio": 0.0, "completion_length": 610.2916793823242, "epoch": 0.30533333333333335, "grad_norm": 0.7943815588951111, "kl": 0.713165283203125, "learning_rate": 2.6309185652526653e-06, "loss": -0.0227, "reward": 0.3333333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 458 }, { "clip_ratio": 0.0, "completion_length": 708.5416946411133, "epoch": 0.306, "grad_norm": 0.45470231771469116, "kl": 0.82373046875, "learning_rate": 2.6286223633435867e-06, "loss": 0.1167, "reward": 0.1666666716337204, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 459 }, { "clip_ratio": 0.0, "completion_length": 800.5000152587891, "epoch": 0.30666666666666664, "grad_norm": 0.5670386552810669, "kl": 0.973876953125, "learning_rate": 2.626320049472249e-06, "loss": 0.0917, "reward": 0.2083333358168602, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.2083333358168602, "rewards/format_reward": 0.0, "step": 460 }, { "clip_ratio": 0.0, "completion_length": 524.5416793823242, "epoch": 0.30733333333333335, "grad_norm": 0.2521005868911743, "kl": 0.4332275390625, "learning_rate": 2.6240116361066445e-06, "loss": 0.0191, "reward": 0.5416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.5416666679084301, "rewards/format_reward": 0.0, "step": 461 }, { "clip_ratio": 0.0, "completion_length": 686.8333435058594, "epoch": 0.308, "grad_norm": 0.5253229141235352, "kl": 0.75830078125, "learning_rate": 2.621697135747798e-06, "loss": 0.1143, "reward": 0.2916666716337204, "reward_std": 0.39512956142425537, "rewards/accuracy_reward": 0.2916666716337204, "rewards/format_reward": 0.0, "step": 462 }, { "clip_ratio": 0.0, "completion_length": 512.8333511352539, "epoch": 0.30866666666666664, "grad_norm": 1.5049155950546265, "kl": 0.49609375, "learning_rate": 2.6193765609296965e-06, "loss": 0.1471, "reward": 0.5416666865348816, "reward_std": 0.3410547897219658, "rewards/accuracy_reward": 0.5416666865348816, "rewards/format_reward": 0.0, "step": 463 }, { "clip_ratio": 0.0, "completion_length": 963.8750152587891, "epoch": 0.30933333333333335, "grad_norm": 0.3934796452522278, "kl": 1.419921875, "learning_rate": 2.6170499242192243e-06, "loss": 0.1396, "reward": 0.1250000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 464 }, { "clip_ratio": 0.0, "completion_length": 801.1666870117188, "epoch": 0.31, "grad_norm": 0.6498653888702393, "kl": 0.87060546875, "learning_rate": 2.6147172382160914e-06, "loss": 0.0634, "reward": 0.2500000074505806, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 465 }, { "clip_ratio": 0.0, "completion_length": 526.5416870117188, "epoch": 0.31066666666666665, "grad_norm": 0.5797969102859497, "kl": 0.369842529296875, "learning_rate": 2.6123785155527693e-06, "loss": 0.2569, "reward": 0.5833333358168602, "reward_std": 0.40296074748039246, "rewards/accuracy_reward": 0.5833333358168602, "rewards/format_reward": 0.0, "step": 466 }, { "clip_ratio": 0.0, "completion_length": 648.7083511352539, "epoch": 0.31133333333333335, "grad_norm": 0.37764111161231995, "kl": 0.56011962890625, "learning_rate": 2.6100337688944195e-06, "loss": 0.0984, "reward": 0.416666679084301, "reward_std": 0.3680921792984009, "rewards/accuracy_reward": 0.416666679084301, "rewards/format_reward": 0.0, "step": 467 }, { "clip_ratio": 0.0, "completion_length": 712.0000152587891, "epoch": 0.312, "grad_norm": 1.0188205242156982, "kl": 0.775390625, "learning_rate": 2.607683010938826e-06, "loss": 0.0639, "reward": 0.4583333469927311, "reward_std": 0.4893604516983032, "rewards/accuracy_reward": 0.4583333469927311, "rewards/format_reward": 0.0, "step": 468 }, { "clip_ratio": 0.0, "completion_length": 519.7916870117188, "epoch": 0.31266666666666665, "grad_norm": 1.1194274425506592, "kl": 0.314208984375, "learning_rate": 2.605326254416326e-06, "loss": 0.0989, "reward": 0.5833333507180214, "reward_std": 0.4971916377544403, "rewards/accuracy_reward": 0.5833333507180214, "rewards/format_reward": 0.0, "step": 469 }, { "clip_ratio": 0.0, "completion_length": 889.1666870117188, "epoch": 0.31333333333333335, "grad_norm": 0.7505325078964233, "kl": 1.607421875, "learning_rate": 2.6029635120897432e-06, "loss": 0.1122, "reward": 0.0416666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 470 }, { "clip_ratio": 0.0, "completion_length": 821.2916870117188, "epoch": 0.314, "grad_norm": 0.6851320862770081, "kl": 1.06640625, "learning_rate": 2.6005947967543145e-06, "loss": 0.1171, "reward": 0.1666666716337204, "reward_std": 0.40824829041957855, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 471 }, { "clip_ratio": 0.0, "completion_length": 523.8333435058594, "epoch": 0.31466666666666665, "grad_norm": 0.42688632011413574, "kl": 0.48602294921875, "learning_rate": 2.5982201212376253e-06, "loss": 0.0579, "reward": 0.2916666716337204, "reward_std": 0.39512956142425537, "rewards/accuracy_reward": 0.2916666716337204, "rewards/format_reward": 0.0, "step": 472 }, { "clip_ratio": 0.0, "completion_length": 853.4166717529297, "epoch": 0.31533333333333335, "grad_norm": 0.353791743516922, "kl": 1.0341796875, "learning_rate": 2.595839498399537e-06, "loss": 0.062, "reward": 0.1250000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 473 }, { "clip_ratio": 0.0, "completion_length": 638.6666870117188, "epoch": 0.316, "grad_norm": 0.20501375198364258, "kl": 0.58984375, "learning_rate": 2.5934529411321173e-06, "loss": 0.0492, "reward": 0.125, "reward_std": 0.1369306445121765, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "step": 474 }, { "clip_ratio": 0.0, "completion_length": 760.5416870117188, "epoch": 0.31666666666666665, "grad_norm": 0.6881753206253052, "kl": 1.2373046875, "learning_rate": 2.591060462359573e-06, "loss": 0.0752, "reward": 0.2500000037252903, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.0, "step": 475 }, { "clip_ratio": 0.0, "completion_length": 840.7083587646484, "epoch": 0.31733333333333336, "grad_norm": 0.17403768002986908, "kl": 0.7841796875, "learning_rate": 2.588662075038178e-06, "loss": 0.0487, "reward": 0.0833333358168602, "reward_std": 0.20412415266036987, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 476 }, { "clip_ratio": 0.0, "completion_length": 576.2916717529297, "epoch": 0.318, "grad_norm": 0.770028829574585, "kl": 0.81982421875, "learning_rate": 2.5862577921562017e-06, "loss": 0.102, "reward": 0.4166666828095913, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.4166666828095913, "rewards/format_reward": 0.0, "step": 477 }, { "clip_ratio": 0.0, "completion_length": 496.0416717529297, "epoch": 0.31866666666666665, "grad_norm": 0.4176042973995209, "kl": 0.30126953125, "learning_rate": 2.583847626733842e-06, "loss": 0.0308, "reward": 0.4583333395421505, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.4583333395421505, "rewards/format_reward": 0.0, "step": 478 }, { "clip_ratio": 0.0, "completion_length": 516.4166793823242, "epoch": 0.31933333333333336, "grad_norm": 0.326368123292923, "kl": 0.5277099609375, "learning_rate": 2.5814315918231546e-06, "loss": 0.0594, "reward": 0.5000000037252903, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.5000000037252903, "rewards/format_reward": 0.0, "step": 479 }, { "clip_ratio": 0.0, "completion_length": 701.0833587646484, "epoch": 0.32, "grad_norm": 0.3699897825717926, "kl": 0.601806640625, "learning_rate": 2.5790097005079765e-06, "loss": 0.1122, "reward": 0.2916666716337204, "reward_std": 0.39512956142425537, "rewards/accuracy_reward": 0.2916666716337204, "rewards/format_reward": 0.0, "step": 480 }, { "clip_ratio": 0.0, "completion_length": 540.0416793823242, "epoch": 0.32066666666666666, "grad_norm": 0.47314339876174927, "kl": 1.001953125, "learning_rate": 2.5765819659038648e-06, "loss": 0.1522, "reward": 0.3333333395421505, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.3333333395421505, "rewards/format_reward": 0.0, "step": 481 }, { "clip_ratio": 0.0, "completion_length": 437.62501525878906, "epoch": 0.32133333333333336, "grad_norm": 0.5493364930152893, "kl": 0.6221923828125, "learning_rate": 2.574148401158017e-06, "loss": 0.1163, "reward": 0.5416666716337204, "reward_std": 0.39512956142425537, "rewards/accuracy_reward": 0.5416666716337204, "rewards/format_reward": 0.0, "step": 482 }, { "clip_ratio": 0.0, "completion_length": 528.6666717529297, "epoch": 0.322, "grad_norm": 1.1089591979980469, "kl": 1.55645751953125, "learning_rate": 2.571709019449205e-06, "loss": 0.1224, "reward": 0.3750000037252903, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.3750000037252903, "rewards/format_reward": 0.0, "step": 483 }, { "clip_ratio": 0.0, "completion_length": 564.8750152587891, "epoch": 0.32266666666666666, "grad_norm": 0.6077021360397339, "kl": 1.33056640625, "learning_rate": 2.5692638339877007e-06, "loss": 0.1831, "reward": 0.5416666753590107, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.5416666753590107, "rewards/format_reward": 0.0, "step": 484 }, { "clip_ratio": 0.0, "completion_length": 559.4166870117188, "epoch": 0.3233333333333333, "grad_norm": 0.7261531352996826, "kl": 0.8671875, "learning_rate": 2.566812858015206e-06, "loss": 0.1036, "reward": 0.3750000111758709, "reward_std": 0.3410547822713852, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 485 }, { "clip_ratio": 0.0, "completion_length": 652.3333587646484, "epoch": 0.324, "grad_norm": 4.562093734741211, "kl": 1.6181640625, "learning_rate": 2.5643561048047816e-06, "loss": 0.0182, "reward": 0.0833333358168602, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 486 }, { "clip_ratio": 0.0, "completion_length": 755.0416870117188, "epoch": 0.32466666666666666, "grad_norm": 0.2521772086620331, "kl": 0.857421875, "learning_rate": 2.561893587660774e-06, "loss": 0.0763, "reward": 0.1250000037252903, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 487 }, { "clip_ratio": 0.0, "completion_length": 604.7500076293945, "epoch": 0.3253333333333333, "grad_norm": 0.7781558036804199, "kl": 1.21435546875, "learning_rate": 2.559425319918743e-06, "loss": 0.1766, "reward": 0.4583333432674408, "reward_std": 0.5242290198802948, "rewards/accuracy_reward": 0.4583333432674408, "rewards/format_reward": 0.0, "step": 488 }, { "clip_ratio": 0.0, "completion_length": 630.3333435058594, "epoch": 0.326, "grad_norm": 0.559182345867157, "kl": 0.83203125, "learning_rate": 2.556951314945392e-06, "loss": 0.038, "reward": 0.2500000074505806, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 489 }, { "clip_ratio": 0.0, "completion_length": 537.3333549499512, "epoch": 0.32666666666666666, "grad_norm": 0.458783358335495, "kl": 0.9066162109375, "learning_rate": 2.5544715861384928e-06, "loss": 0.1204, "reward": 0.4166666753590107, "reward_std": 0.4971916377544403, "rewards/accuracy_reward": 0.4166666753590107, "rewards/format_reward": 0.0, "step": 490 }, { "clip_ratio": 0.0, "completion_length": 654.0000152587891, "epoch": 0.3273333333333333, "grad_norm": 0.3278976082801819, "kl": 0.96954345703125, "learning_rate": 2.5519861469268135e-06, "loss": 0.042, "reward": 0.2500000074505806, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 491 }, { "clip_ratio": 0.0, "completion_length": 681.0000152587891, "epoch": 0.328, "grad_norm": 0.6427708268165588, "kl": 1.8046875, "learning_rate": 2.549495010770048e-06, "loss": 0.3858, "reward": 0.3333333432674408, "reward_std": 0.3872983753681183, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 492 }, { "clip_ratio": 0.0, "completion_length": 624.8750305175781, "epoch": 0.32866666666666666, "grad_norm": 0.8548335433006287, "kl": 1.4541015625, "learning_rate": 2.546998191158741e-06, "loss": 0.206, "reward": 0.2083333358168602, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.2083333358168602, "rewards/format_reward": 0.0, "step": 493 }, { "clip_ratio": 0.0, "completion_length": 577.0833511352539, "epoch": 0.3293333333333333, "grad_norm": 0.4680027663707733, "kl": 0.714752197265625, "learning_rate": 2.5444957016142144e-06, "loss": 0.0329, "reward": 0.5833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.5833333358168602, "rewards/format_reward": 0.0, "step": 494 }, { "clip_ratio": 0.0, "completion_length": 554.2083358764648, "epoch": 0.33, "grad_norm": 1.1668338775634766, "kl": 1.142333984375, "learning_rate": 2.5419875556884957e-06, "loss": 0.197, "reward": 0.3333333395421505, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.3333333395421505, "rewards/format_reward": 0.0, "step": 495 }, { "clip_ratio": 0.0, "completion_length": 558.1666946411133, "epoch": 0.33066666666666666, "grad_norm": 0.5063526630401611, "kl": 0.95257568359375, "learning_rate": 2.5394737669642457e-06, "loss": 0.043, "reward": 0.25, "reward_std": 0.0, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "step": 496 }, { "clip_ratio": 0.0, "completion_length": 498.5416793823242, "epoch": 0.3313333333333333, "grad_norm": 0.29962876439094543, "kl": 0.71697998046875, "learning_rate": 2.5369543490546814e-06, "loss": 0.0998, "reward": 0.5000000223517418, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.5000000223517418, "rewards/format_reward": 0.0, "step": 497 }, { "clip_ratio": 0.0, "completion_length": 668.5000305175781, "epoch": 0.332, "grad_norm": 0.6125059127807617, "kl": 1.1259765625, "learning_rate": 2.5344293156035046e-06, "loss": 0.0529, "reward": 0.0833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 498 }, { "clip_ratio": 0.0, "completion_length": 706.0, "epoch": 0.33266666666666667, "grad_norm": 1.2055842876434326, "kl": 1.439208984375, "learning_rate": 2.5318986802848277e-06, "loss": 0.1032, "reward": 0.125, "reward_std": 0.1369306445121765, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "step": 499 }, { "clip_ratio": 0.0, "completion_length": 553.0416793823242, "epoch": 0.3333333333333333, "grad_norm": 0.3650849163532257, "kl": 0.8133544921875, "learning_rate": 2.529362456803101e-06, "loss": 0.0829, "reward": 0.4583333395421505, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.4583333395421505, "rewards/format_reward": 0.0, "step": 500 }, { "clip_ratio": 0.0, "completion_length": 590.2083511352539, "epoch": 0.334, "grad_norm": 0.26055192947387695, "kl": 0.39959716796875, "learning_rate": 2.526820658893033e-06, "loss": 0.0339, "reward": 0.5, "reward_std": 0.273861289024353, "rewards/accuracy_reward": 0.5, "rewards/format_reward": 0.0, "step": 501 }, { "clip_ratio": 0.0, "completion_length": 402.16667556762695, "epoch": 0.33466666666666667, "grad_norm": 0.37195783853530884, "kl": 0.17681884765625, "learning_rate": 2.5242733003195252e-06, "loss": 0.0431, "reward": 0.5416666753590107, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.5416666753590107, "rewards/format_reward": 0.0, "step": 502 }, { "clip_ratio": 0.0, "completion_length": 754.2083587646484, "epoch": 0.3353333333333333, "grad_norm": 0.4035155773162842, "kl": 0.556396484375, "learning_rate": 2.521720394877589e-06, "loss": 0.0889, "reward": 0.1666666679084301, "reward_std": 0.23899272084236145, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 503 }, { "clip_ratio": 0.0, "completion_length": 446.3750228881836, "epoch": 0.336, "grad_norm": 0.7838923931121826, "kl": 0.34844970703125, "learning_rate": 2.519161956392275e-06, "loss": 0.0954, "reward": 0.541666679084301, "reward_std": 0.306186206638813, "rewards/accuracy_reward": 0.541666679084301, "rewards/format_reward": 0.0, "step": 504 }, { "clip_ratio": 0.0, "completion_length": 498.3750114440918, "epoch": 0.33666666666666667, "grad_norm": 0.21071508526802063, "kl": 0.246002197265625, "learning_rate": 2.516597998718599e-06, "loss": 0.0488, "reward": 0.3333333358168602, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 505 }, { "clip_ratio": 0.0, "completion_length": 419.50000762939453, "epoch": 0.3373333333333333, "grad_norm": 0.6493383646011353, "kl": 0.245361328125, "learning_rate": 2.514028535741463e-06, "loss": 0.054, "reward": 0.6666666828095913, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.6666666828095913, "rewards/format_reward": 0.0, "step": 506 }, { "clip_ratio": 0.0, "completion_length": 699.1666946411133, "epoch": 0.338, "grad_norm": 0.3074341416358948, "kl": 0.68646240234375, "learning_rate": 2.511453581375585e-06, "loss": 0.0527, "reward": 0.2083333432674408, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.2083333432674408, "rewards/format_reward": 0.0, "step": 507 }, { "clip_ratio": 0.0, "completion_length": 745.1250152587891, "epoch": 0.33866666666666667, "grad_norm": 0.672670841217041, "kl": 0.5244140625, "learning_rate": 2.5088731495654205e-06, "loss": 0.1203, "reward": 0.2500000111758709, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.2500000111758709, "rewards/format_reward": 0.0, "step": 508 }, { "clip_ratio": 0.0, "completion_length": 475.3750228881836, "epoch": 0.3393333333333333, "grad_norm": 0.2755073606967926, "kl": 0.3074951171875, "learning_rate": 2.5062872542850865e-06, "loss": 0.0737, "reward": 0.4166666716337204, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.4166666716337204, "rewards/format_reward": 0.0, "step": 509 }, { "clip_ratio": 0.0, "completion_length": 545.4583435058594, "epoch": 0.34, "grad_norm": 0.3922642171382904, "kl": 0.5213623046875, "learning_rate": 2.5036959095382875e-06, "loss": -0.0644, "reward": 0.416666679084301, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.416666679084301, "rewards/format_reward": 0.0, "step": 510 }, { "clip_ratio": 0.0, "completion_length": 705.2083435058594, "epoch": 0.3406666666666667, "grad_norm": 0.28274038434028625, "kl": 0.937255859375, "learning_rate": 2.5010991293582397e-06, "loss": 0.0736, "reward": 0.0416666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 511 }, { "clip_ratio": 0.0, "completion_length": 447.1250228881836, "epoch": 0.3413333333333333, "grad_norm": 0.24494610726833344, "kl": 0.150634765625, "learning_rate": 2.4984969278075954e-06, "loss": 0.0201, "reward": 0.3333333432674408, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 512 }, { "clip_ratio": 0.0, "completion_length": 590.5833511352539, "epoch": 0.342, "grad_norm": 1.1417653560638428, "kl": 1.40234375, "learning_rate": 2.4958893189783626e-06, "loss": 0.0725, "reward": 0.2083333395421505, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 513 }, { "clip_ratio": 0.0, "completion_length": 672.5416946411133, "epoch": 0.3426666666666667, "grad_norm": 1.132948637008667, "kl": 0.593017578125, "learning_rate": 2.4932763169918353e-06, "loss": 0.0866, "reward": 0.5000000149011612, "reward_std": 0.4971916303038597, "rewards/accuracy_reward": 0.5000000149011612, "rewards/format_reward": 0.0, "step": 514 }, { "clip_ratio": 0.0, "completion_length": 740.7916870117188, "epoch": 0.3433333333333333, "grad_norm": 2.203300952911377, "kl": 0.841796875, "learning_rate": 2.4906579359985114e-06, "loss": 0.0444, "reward": 0.4583333469927311, "reward_std": 0.4701542407274246, "rewards/accuracy_reward": 0.4583333469927311, "rewards/format_reward": 0.0, "step": 515 }, { "clip_ratio": 0.0, "completion_length": 668.375, "epoch": 0.344, "grad_norm": 0.5682621598243713, "kl": 0.593505859375, "learning_rate": 2.4880341901780208e-06, "loss": 0.0439, "reward": 0.2916666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.2916666679084301, "rewards/format_reward": 0.0, "step": 516 }, { "clip_ratio": 0.0, "completion_length": 623.5416870117188, "epoch": 0.3446666666666667, "grad_norm": 0.8462840914726257, "kl": 0.37738037109375, "learning_rate": 2.485405093739044e-06, "loss": 0.1223, "reward": 0.5416666902601719, "reward_std": 0.43528566509485245, "rewards/accuracy_reward": 0.5416666902601719, "rewards/format_reward": 0.0, "step": 517 }, { "clip_ratio": 0.0, "completion_length": 563.6250114440918, "epoch": 0.3453333333333333, "grad_norm": 0.5932950973510742, "kl": 0.94537353515625, "learning_rate": 2.4827706609192375e-06, "loss": 0.0242, "reward": 0.2916666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.2916666679084301, "rewards/format_reward": 0.0, "step": 518 }, { "clip_ratio": 0.0, "completion_length": 770.0833435058594, "epoch": 0.346, "grad_norm": 0.9637057781219482, "kl": 1.376953125, "learning_rate": 2.4801309059851586e-06, "loss": 0.1484, "reward": 0.2083333395421505, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 519 }, { "clip_ratio": 0.0, "completion_length": 699.5833587646484, "epoch": 0.3466666666666667, "grad_norm": 1.1205428838729858, "kl": 1.3779296875, "learning_rate": 2.477485843232183e-06, "loss": 0.0124, "reward": 0.0416666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 520 }, { "clip_ratio": 0.0, "completion_length": 757.6250305175781, "epoch": 0.3473333333333333, "grad_norm": 0.4129210412502289, "kl": 0.67919921875, "learning_rate": 2.474835486984433e-06, "loss": 0.0599, "reward": 0.1666666679084301, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 521 }, { "clip_ratio": 0.0, "completion_length": 705.4583435058594, "epoch": 0.348, "grad_norm": 0.47046324610710144, "kl": 1.05517578125, "learning_rate": 2.4721798515946964e-06, "loss": 0.1536, "reward": 0.2083333395421505, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 522 }, { "clip_ratio": 0.0, "completion_length": 487.375, "epoch": 0.3486666666666667, "grad_norm": 1.626871943473816, "kl": 0.37322998046875, "learning_rate": 2.4695189514443497e-06, "loss": 0.2106, "reward": 0.5833333432674408, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.5833333432674408, "rewards/format_reward": 0.0, "step": 523 }, { "clip_ratio": 0.0, "completion_length": 486.2916717529297, "epoch": 0.34933333333333333, "grad_norm": 1.9238725900650024, "kl": 0.61865234375, "learning_rate": 2.4668528009432804e-06, "loss": 0.2909, "reward": 0.541666679084301, "reward_std": 0.5242290198802948, "rewards/accuracy_reward": 0.541666679084301, "rewards/format_reward": 0.0, "step": 524 }, { "clip_ratio": 0.0, "completion_length": 643.0416793823242, "epoch": 0.35, "grad_norm": 0.4621191918849945, "kl": 0.298828125, "learning_rate": 2.464181414529809e-06, "loss": 0.0214, "reward": 0.5000000186264515, "reward_std": 0.46232306212186813, "rewards/accuracy_reward": 0.5000000186264515, "rewards/format_reward": 0.0, "step": 525 }, { "clip_ratio": 0.0, "completion_length": 733.4166870117188, "epoch": 0.3506666666666667, "grad_norm": 0.5585741400718689, "kl": 0.24224853515625, "learning_rate": 2.4615048066706103e-06, "loss": 0.0286, "reward": 0.2916666753590107, "reward_std": 0.43528567254543304, "rewards/accuracy_reward": 0.2916666753590107, "rewards/format_reward": 0.0, "step": 526 }, { "clip_ratio": 0.0, "completion_length": 513.1666793823242, "epoch": 0.35133333333333333, "grad_norm": 1.475561261177063, "kl": 0.322021484375, "learning_rate": 2.4588229918606356e-06, "loss": 0.1456, "reward": 0.5416666902601719, "reward_std": 0.43528567254543304, "rewards/accuracy_reward": 0.5416666902601719, "rewards/format_reward": 0.0, "step": 527 }, { "clip_ratio": 0.0, "completion_length": 527.0833511352539, "epoch": 0.352, "grad_norm": 0.6907888054847717, "kl": 0.6083984375, "learning_rate": 2.456135984623035e-06, "loss": 0.0519, "reward": 0.291666679084301, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.291666679084301, "rewards/format_reward": 0.0, "step": 528 }, { "clip_ratio": 0.0, "completion_length": 537.4166831970215, "epoch": 0.3526666666666667, "grad_norm": 1.4600276947021484, "kl": 1.53662109375, "learning_rate": 2.453443799509075e-06, "loss": 0.1233, "reward": 0.3333333358168602, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 529 }, { "clip_ratio": 0.0, "completion_length": 544.6250190734863, "epoch": 0.35333333333333333, "grad_norm": 3.3162546157836914, "kl": 2.16015625, "learning_rate": 2.4507464510980654e-06, "loss": 0.1042, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 530 }, { "clip_ratio": 0.0, "completion_length": 675.1250152587891, "epoch": 0.354, "grad_norm": 1.340352177619934, "kl": 2.09423828125, "learning_rate": 2.4480439539972765e-06, "loss": 0.059, "reward": 0.2083333395421505, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 531 }, { "clip_ratio": 0.0, "completion_length": 750.3750152587891, "epoch": 0.3546666666666667, "grad_norm": 8.773200988769531, "kl": 2.1064453125, "learning_rate": 2.44533632284186e-06, "loss": 0.1378, "reward": 0.0833333358168602, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 532 }, { "clip_ratio": 0.0, "completion_length": 530.2083511352539, "epoch": 0.35533333333333333, "grad_norm": 0.5800600051879883, "kl": 1.5103759765625, "learning_rate": 2.4426235722947718e-06, "loss": 0.209, "reward": 0.5000000111758709, "reward_std": 0.20412415266036987, "rewards/accuracy_reward": 0.5000000111758709, "rewards/format_reward": 0.0, "step": 533 }, { "clip_ratio": 0.0, "completion_length": 589.2083511352539, "epoch": 0.356, "grad_norm": 0.3369172513484955, "kl": 0.766357421875, "learning_rate": 2.439905717046691e-06, "loss": 0.1242, "reward": 0.2500000037252903, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.0, "step": 534 }, { "clip_ratio": 0.0, "completion_length": 623.4583511352539, "epoch": 0.3566666666666667, "grad_norm": 1.1922731399536133, "kl": 0.5208740234375, "learning_rate": 2.43718277181594e-06, "loss": 0.1376, "reward": 0.3750000074505806, "reward_std": 0.4701542481780052, "rewards/accuracy_reward": 0.3750000074505806, "rewards/format_reward": 0.0, "step": 535 }, { "clip_ratio": 0.0, "completion_length": 579.2916870117188, "epoch": 0.35733333333333334, "grad_norm": 0.8067518472671509, "kl": 0.83837890625, "learning_rate": 2.434454751348408e-06, "loss": 0.1387, "reward": 0.541666679084301, "reward_std": 0.5050228163599968, "rewards/accuracy_reward": 0.541666679084301, "rewards/format_reward": 0.0, "step": 536 }, { "clip_ratio": 0.0, "completion_length": 699.9583511352539, "epoch": 0.358, "grad_norm": 1.202216386795044, "kl": 0.98828125, "learning_rate": 2.4317216704174657e-06, "loss": 0.1316, "reward": 0.2916666679084301, "reward_std": 0.37592335790395737, "rewards/accuracy_reward": 0.2916666679084301, "rewards/format_reward": 0.0, "step": 537 }, { "clip_ratio": 0.0, "completion_length": 861.2083587646484, "epoch": 0.3586666666666667, "grad_norm": 0.5167230367660522, "kl": 2.0576171875, "learning_rate": 2.4289835438238904e-06, "loss": 0.2187, "reward": 0.291666679084301, "reward_std": 0.3061862215399742, "rewards/accuracy_reward": 0.291666679084301, "rewards/format_reward": 0.0, "step": 538 }, { "clip_ratio": 0.0, "completion_length": 635.2083435058594, "epoch": 0.35933333333333334, "grad_norm": 1.1617324352264404, "kl": 2.228515625, "learning_rate": 2.4262403863957824e-06, "loss": 0.1695, "reward": 0.1666666716337204, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 539 }, { "clip_ratio": 0.0, "completion_length": 698.0833435058594, "epoch": 0.36, "grad_norm": 1.8295402526855469, "kl": 2.404296875, "learning_rate": 2.4234922129884873e-06, "loss": 0.1678, "reward": 0.2500000074505806, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 540 }, { "clip_ratio": 0.0, "completion_length": 740.5000228881836, "epoch": 0.3606666666666667, "grad_norm": 2.3800482749938965, "kl": 1.943115234375, "learning_rate": 2.420739038484514e-06, "loss": 0.0846, "reward": 0.25, "reward_std": 0.0, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "step": 541 }, { "clip_ratio": 0.0, "completion_length": 617.9583511352539, "epoch": 0.36133333333333334, "grad_norm": 0.8401850461959839, "kl": 1.0740966796875, "learning_rate": 2.417980877793454e-06, "loss": 0.0296, "reward": 0.3333333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 542 }, { "clip_ratio": 0.0, "completion_length": 736.7916870117188, "epoch": 0.362, "grad_norm": 0.7261059880256653, "kl": 1.07421875, "learning_rate": 2.4152177458519018e-06, "loss": 0.0958, "reward": 0.1666666679084301, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 543 }, { "clip_ratio": 0.0, "completion_length": 414.62501525878906, "epoch": 0.3626666666666667, "grad_norm": 0.1728622019290924, "kl": 0.2861328125, "learning_rate": 2.4124496576233714e-06, "loss": 0.0129, "reward": 0.5, "reward_std": 0.0, "rewards/accuracy_reward": 0.5, "rewards/format_reward": 0.0, "step": 544 }, { "clip_ratio": 0.0, "completion_length": 520.8750152587891, "epoch": 0.36333333333333334, "grad_norm": 0.3724670708179474, "kl": 0.27685546875, "learning_rate": 2.4096766280982206e-06, "loss": 0.0753, "reward": 0.4583333395421505, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.4583333395421505, "rewards/format_reward": 0.0, "step": 545 }, { "clip_ratio": 0.0, "completion_length": 731.8333435058594, "epoch": 0.364, "grad_norm": 0.3591066002845764, "kl": 0.53173828125, "learning_rate": 2.4068986722935626e-06, "loss": 0.0269, "reward": 0.0833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 546 }, { "clip_ratio": 0.0, "completion_length": 616.5416793823242, "epoch": 0.36466666666666664, "grad_norm": 0.9663143754005432, "kl": 0.4599609375, "learning_rate": 2.4041158052531905e-06, "loss": 0.1424, "reward": 0.4583333507180214, "reward_std": 0.4893604442477226, "rewards/accuracy_reward": 0.4583333507180214, "rewards/format_reward": 0.0, "step": 547 }, { "clip_ratio": 0.0, "completion_length": 580.6250152587891, "epoch": 0.36533333333333334, "grad_norm": 0.3608686029911041, "kl": 0.57080078125, "learning_rate": 2.4013280420474953e-06, "loss": 0.1086, "reward": 0.2083333358168602, "reward_std": 0.3410547822713852, "rewards/accuracy_reward": 0.2083333358168602, "rewards/format_reward": 0.0, "step": 548 }, { "clip_ratio": 0.0, "completion_length": 716.5416717529297, "epoch": 0.366, "grad_norm": 0.13290414214134216, "kl": 0.38153076171875, "learning_rate": 2.3985353977733787e-06, "loss": 0.0555, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 549 }, { "clip_ratio": 0.0, "completion_length": 743.3333549499512, "epoch": 0.36666666666666664, "grad_norm": 0.21814502775669098, "kl": 0.65277099609375, "learning_rate": 2.3957378875541795e-06, "loss": 0.0284, "reward": 0.25, "reward_std": 0.0, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "step": 550 }, { "clip_ratio": 0.0, "completion_length": 483.5833435058594, "epoch": 0.36733333333333335, "grad_norm": 0.16194969415664673, "kl": 0.24859619140625, "learning_rate": 2.3929355265395845e-06, "loss": 0.0384, "reward": 0.5833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.5833333358168602, "rewards/format_reward": 0.0, "step": 551 }, { "clip_ratio": 0.0, "completion_length": 506.79168701171875, "epoch": 0.368, "grad_norm": 0.49630460143089294, "kl": 0.23724365234375, "learning_rate": 2.3901283299055523e-06, "loss": 0.1289, "reward": 0.6250000111758709, "reward_std": 0.3410547897219658, "rewards/accuracy_reward": 0.6250000111758709, "rewards/format_reward": 0.0, "step": 552 }, { "clip_ratio": 0.0, "completion_length": 766.6250152587891, "epoch": 0.36866666666666664, "grad_norm": 0.2723495066165924, "kl": 0.771484375, "learning_rate": 2.387316312854226e-06, "loss": 0.0892, "reward": 0.1250000037252903, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 553 }, { "clip_ratio": 0.0, "completion_length": 447.7916793823242, "epoch": 0.36933333333333335, "grad_norm": 0.3350905478000641, "kl": 0.18463134765625, "learning_rate": 2.3844994906138548e-06, "loss": 0.0272, "reward": 0.3750000111758709, "reward_std": 0.3410547971725464, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 554 }, { "clip_ratio": 0.0, "completion_length": 601.1666870117188, "epoch": 0.37, "grad_norm": 0.450183242559433, "kl": 0.693359375, "learning_rate": 2.3816778784387097e-06, "loss": 0.1138, "reward": 0.3750000111758709, "reward_std": 0.3410547897219658, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 555 }, { "clip_ratio": 0.0, "completion_length": 609.0000076293945, "epoch": 0.37066666666666664, "grad_norm": 0.21433468163013458, "kl": 0.6109619140625, "learning_rate": 2.3788514916090007e-06, "loss": 0.081, "reward": 0.4166666716337204, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.4166666716337204, "rewards/format_reward": 0.0, "step": 556 }, { "clip_ratio": 0.0, "completion_length": 639.7500228881836, "epoch": 0.37133333333333335, "grad_norm": 0.3514927923679352, "kl": 0.534423828125, "learning_rate": 2.3760203454307957e-06, "loss": 0.0782, "reward": 0.2083333432674408, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.2083333432674408, "rewards/format_reward": 0.0, "step": 557 }, { "clip_ratio": 0.0, "completion_length": 426.58335876464844, "epoch": 0.372, "grad_norm": 0.7129497528076172, "kl": 0.400390625, "learning_rate": 2.3731844552359343e-06, "loss": 0.1745, "reward": 0.7083333432674408, "reward_std": 0.39512956142425537, "rewards/accuracy_reward": 0.7083333432674408, "rewards/format_reward": 0.0, "step": 558 }, { "clip_ratio": 0.0, "completion_length": 485.0416717529297, "epoch": 0.37266666666666665, "grad_norm": 0.4279105067253113, "kl": 0.59576416015625, "learning_rate": 2.3703438363819494e-06, "loss": 0.0983, "reward": 0.3750000111758709, "reward_std": 0.3410547971725464, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 559 }, { "clip_ratio": 0.0, "completion_length": 551.9583587646484, "epoch": 0.37333333333333335, "grad_norm": 0.4485958218574524, "kl": 0.503662109375, "learning_rate": 2.36749850425198e-06, "loss": 0.0738, "reward": 0.2500000037252903, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.0, "step": 560 }, { "clip_ratio": 0.0, "completion_length": 536.5416870117188, "epoch": 0.374, "grad_norm": 0.3974195122718811, "kl": 0.593505859375, "learning_rate": 2.36464847425469e-06, "loss": 0.1405, "reward": 0.4166666679084301, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.4166666679084301, "rewards/format_reward": 0.0, "step": 561 }, { "clip_ratio": 0.0, "completion_length": 641.3333435058594, "epoch": 0.37466666666666665, "grad_norm": 0.24328310787677765, "kl": 0.44708251953125, "learning_rate": 2.3617937618241844e-06, "loss": 0.0644, "reward": 0.3750000149011612, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.3750000149011612, "rewards/format_reward": 0.0, "step": 562 }, { "clip_ratio": 0.0, "completion_length": 641.1250152587891, "epoch": 0.37533333333333335, "grad_norm": 0.6015921831130981, "kl": 0.46807861328125, "learning_rate": 2.358934382419925e-06, "loss": 0.1122, "reward": 0.4583333469927311, "reward_std": 0.4701542407274246, "rewards/accuracy_reward": 0.4583333469927311, "rewards/format_reward": 0.0, "step": 563 }, { "clip_ratio": 0.0, "completion_length": 602.7916717529297, "epoch": 0.376, "grad_norm": 0.3574065566062927, "kl": 0.422576904296875, "learning_rate": 2.356070351526648e-06, "loss": 0.1686, "reward": 0.4166666679084301, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.4166666679084301, "rewards/format_reward": 0.0, "step": 564 }, { "clip_ratio": 0.0, "completion_length": 774.3333587646484, "epoch": 0.37666666666666665, "grad_norm": 0.281559020280838, "kl": 1.11083984375, "learning_rate": 2.353201684654278e-06, "loss": 0.0615, "reward": 0.1250000037252903, "reward_std": 0.306186206638813, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 565 }, { "clip_ratio": 0.0, "completion_length": 587.1250228881836, "epoch": 0.37733333333333335, "grad_norm": 0.365109920501709, "kl": 0.220001220703125, "learning_rate": 2.3503283973378465e-06, "loss": -0.0003, "reward": 0.2083333395421505, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 566 }, { "clip_ratio": 0.0, "completion_length": 543.4583473205566, "epoch": 0.378, "grad_norm": 0.5559046268463135, "kl": 0.80621337890625, "learning_rate": 2.347450505137407e-06, "loss": 0.1135, "reward": 0.5000000223517418, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.5000000223517418, "rewards/format_reward": 0.0, "step": 567 }, { "clip_ratio": 0.0, "completion_length": 660.1666870117188, "epoch": 0.37866666666666665, "grad_norm": 0.4049297273159027, "kl": 0.746856689453125, "learning_rate": 2.344568023637949e-06, "loss": 0.1303, "reward": 0.2083333358168602, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.2083333358168602, "rewards/format_reward": 0.0, "step": 568 }, { "clip_ratio": 0.0, "completion_length": 540.5833358764648, "epoch": 0.37933333333333336, "grad_norm": 0.6145519018173218, "kl": 0.7515869140625, "learning_rate": 2.3416809684493166e-06, "loss": 0.1536, "reward": 0.3750000111758709, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 569 }, { "clip_ratio": 0.0, "completion_length": 408.6666793823242, "epoch": 0.38, "grad_norm": 0.9195656776428223, "kl": 0.664642333984375, "learning_rate": 2.3387893552061204e-06, "loss": 0.2207, "reward": 0.8333333432674408, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.8333333432674408, "rewards/format_reward": 0.0, "step": 570 }, { "clip_ratio": 0.0, "completion_length": 480.87500762939453, "epoch": 0.38066666666666665, "grad_norm": 0.8276476860046387, "kl": 1.0643310546875, "learning_rate": 2.335893199567657e-06, "loss": 0.1052, "reward": 0.0833333358168602, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 571 }, { "clip_ratio": 0.0, "completion_length": 810.4583587646484, "epoch": 0.38133333333333336, "grad_norm": 1.4429028034210205, "kl": 2.4658203125, "learning_rate": 2.332992517217819e-06, "loss": 0.1879, "reward": 0.1666666679084301, "reward_std": 0.23899272084236145, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 572 }, { "clip_ratio": 0.0, "completion_length": 708.5833587646484, "epoch": 0.382, "grad_norm": 2.071645498275757, "kl": 2.034423828125, "learning_rate": 2.330087323865016e-06, "loss": 0.0983, "reward": 0.2500000037252903, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.0, "step": 573 }, { "clip_ratio": 0.0, "completion_length": 614.1666870117188, "epoch": 0.38266666666666665, "grad_norm": 0.9461411833763123, "kl": 2.0595703125, "learning_rate": 2.327177635242086e-06, "loss": 0.2466, "reward": 0.3333333358168602, "reward_std": 0.4779854267835617, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 574 }, { "clip_ratio": 0.0, "completion_length": 565.7083435058594, "epoch": 0.38333333333333336, "grad_norm": 0.8399139046669006, "kl": 0.529296875, "learning_rate": 2.324263467106209e-06, "loss": 0.1587, "reward": 0.4166666865348816, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.4166666865348816, "rewards/format_reward": 0.0, "step": 575 }, { "clip_ratio": 0.0, "completion_length": 621.5000152587891, "epoch": 0.384, "grad_norm": 0.33386820554733276, "kl": 0.5238037109375, "learning_rate": 2.3213448352388254e-06, "loss": 0.0831, "reward": 0.2916666753590107, "reward_std": 0.3602609932422638, "rewards/accuracy_reward": 0.2916666753590107, "rewards/format_reward": 0.0, "step": 576 }, { "clip_ratio": 0.0, "completion_length": 678.5833587646484, "epoch": 0.38466666666666666, "grad_norm": 0.45198026299476624, "kl": 0.91357421875, "learning_rate": 2.3184217554455477e-06, "loss": 0.0402, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 577 }, { "clip_ratio": 0.0, "completion_length": 250.75000381469727, "epoch": 0.38533333333333336, "grad_norm": 0.6509225368499756, "kl": 0.34112548828125, "learning_rate": 2.315494243556075e-06, "loss": 0.0348, "reward": 0.833333358168602, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.833333358168602, "rewards/format_reward": 0.0, "step": 578 }, { "clip_ratio": 0.0, "completion_length": 621.0833511352539, "epoch": 0.386, "grad_norm": 0.4850148558616638, "kl": 0.84356689453125, "learning_rate": 2.31256231542411e-06, "loss": 0.0138, "reward": 0.3333333469927311, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.3333333469927311, "rewards/format_reward": 0.0, "step": 579 }, { "clip_ratio": 0.0, "completion_length": 636.5000152587891, "epoch": 0.38666666666666666, "grad_norm": 0.3733702600002289, "kl": 0.82470703125, "learning_rate": 2.3096259869272697e-06, "loss": 0.0379, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 580 }, { "clip_ratio": 0.0, "completion_length": 519.9583549499512, "epoch": 0.3873333333333333, "grad_norm": 0.27327096462249756, "kl": 0.4571533203125, "learning_rate": 2.306685273967001e-06, "loss": 0.1132, "reward": 0.3333333469927311, "reward_std": 0.40824829041957855, "rewards/accuracy_reward": 0.3333333469927311, "rewards/format_reward": 0.0, "step": 581 }, { "clip_ratio": 0.0, "completion_length": 551.8750076293945, "epoch": 0.388, "grad_norm": 0.3963361978530884, "kl": 0.64373779296875, "learning_rate": 2.303740192468495e-06, "loss": 0.1196, "reward": 0.3333333469927311, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.3333333469927311, "rewards/format_reward": 0.0, "step": 582 }, { "clip_ratio": 0.0, "completion_length": 268.58333587646484, "epoch": 0.38866666666666666, "grad_norm": 0.25323012471199036, "kl": 0.05401611328125, "learning_rate": 2.3007907583806005e-06, "loss": 0.0064, "reward": 0.75, "reward_std": 0.273861289024353, "rewards/accuracy_reward": 0.75, "rewards/format_reward": 0.0, "step": 583 }, { "clip_ratio": 0.0, "completion_length": 512.7083511352539, "epoch": 0.3893333333333333, "grad_norm": 0.40740954875946045, "kl": 0.6710205078125, "learning_rate": 2.2978369876757365e-06, "loss": 0.074, "reward": 0.5000000037252903, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.5000000037252903, "rewards/format_reward": 0.0, "step": 584 }, { "clip_ratio": 0.0, "completion_length": 639.4583435058594, "epoch": 0.39, "grad_norm": 0.9055688381195068, "kl": 0.52899169921875, "learning_rate": 2.2948788963498076e-06, "loss": 0.0807, "reward": 0.3750000037252903, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.3750000037252903, "rewards/format_reward": 0.0, "step": 585 }, { "clip_ratio": 0.0, "completion_length": 616.7916717529297, "epoch": 0.39066666666666666, "grad_norm": 0.29027146100997925, "kl": 0.57568359375, "learning_rate": 2.2919165004221152e-06, "loss": 0.1041, "reward": 0.2083333395421505, "reward_std": 0.3602609932422638, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 586 }, { "clip_ratio": 0.0, "completion_length": 690.8750152587891, "epoch": 0.3913333333333333, "grad_norm": 0.7485983967781067, "kl": 1.000732421875, "learning_rate": 2.2889498159352732e-06, "loss": 0.1414, "reward": 0.2916666753590107, "reward_std": 0.43528567254543304, "rewards/accuracy_reward": 0.2916666753590107, "rewards/format_reward": 0.0, "step": 587 }, { "clip_ratio": 0.0, "completion_length": 697.0000152587891, "epoch": 0.392, "grad_norm": 0.7230675220489502, "kl": 1.474609375, "learning_rate": 2.285978858955119e-06, "loss": 0.2686, "reward": 0.2916666753590107, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.2916666753590107, "rewards/format_reward": 0.0, "step": 588 }, { "clip_ratio": 0.0, "completion_length": 536.8750152587891, "epoch": 0.39266666666666666, "grad_norm": 0.5188837647438049, "kl": 1.396484375, "learning_rate": 2.283003645570627e-06, "loss": 0.0612, "reward": 0.1666666716337204, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 589 }, { "clip_ratio": 0.0, "completion_length": 524.2916717529297, "epoch": 0.3933333333333333, "grad_norm": 0.33348211646080017, "kl": 0.75531005859375, "learning_rate": 2.280024191893823e-06, "loss": 0.1091, "reward": 0.3750000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.3750000037252903, "rewards/format_reward": 0.0, "step": 590 }, { "clip_ratio": 0.0, "completion_length": 841.4166870117188, "epoch": 0.394, "grad_norm": 0.40513989329338074, "kl": 0.603271484375, "learning_rate": 2.2770405140596957e-06, "loss": 0.0461, "reward": 0.1666666679084301, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 591 }, { "clip_ratio": 0.0, "completion_length": 530.3333435058594, "epoch": 0.39466666666666667, "grad_norm": 0.7539398670196533, "kl": 0.475921630859375, "learning_rate": 2.274052628226107e-06, "loss": 0.1091, "reward": 0.6666666865348816, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.6666666865348816, "rewards/format_reward": 0.0, "step": 592 }, { "clip_ratio": 0.0, "completion_length": 546.3333358764648, "epoch": 0.3953333333333333, "grad_norm": 0.3055289685726166, "kl": 0.830322265625, "learning_rate": 2.2710605505737107e-06, "loss": 0.1056, "reward": 0.0416666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 593 }, { "clip_ratio": 0.0, "completion_length": 568.5833511352539, "epoch": 0.396, "grad_norm": 2.0718793869018555, "kl": 0.77130126953125, "learning_rate": 2.268064297305857e-06, "loss": 0.0285, "reward": 0.291666679084301, "reward_std": 0.3061862140893936, "rewards/accuracy_reward": 0.291666679084301, "rewards/format_reward": 0.0, "step": 594 }, { "clip_ratio": 0.0, "completion_length": 600.2083587646484, "epoch": 0.39666666666666667, "grad_norm": 0.4746382236480713, "kl": 0.880859375, "learning_rate": 2.265063884648513e-06, "loss": 0.1444, "reward": 0.4166666753590107, "reward_std": 0.4971916377544403, "rewards/accuracy_reward": 0.4166666753590107, "rewards/format_reward": 0.0, "step": 595 }, { "clip_ratio": 0.0, "completion_length": 772.5000152587891, "epoch": 0.3973333333333333, "grad_norm": 0.7006262540817261, "kl": 1.4287109375, "learning_rate": 2.2620593288501667e-06, "loss": 0.066, "reward": 0.1250000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 596 }, { "clip_ratio": 0.0, "completion_length": 623.0000152587891, "epoch": 0.398, "grad_norm": 0.47483310103416443, "kl": 1.7891845703125, "learning_rate": 2.2590506461817453e-06, "loss": 0.1354, "reward": 0.3333333358168602, "reward_std": 0.40296074748039246, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 597 }, { "clip_ratio": 0.0, "completion_length": 652.3333587646484, "epoch": 0.39866666666666667, "grad_norm": 1.4641038179397583, "kl": 2.349609375, "learning_rate": 2.256037852936525e-06, "loss": 0.1695, "reward": 0.1666666716337204, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 598 }, { "clip_ratio": 0.0, "completion_length": 474.0000114440918, "epoch": 0.3993333333333333, "grad_norm": 0.5832303762435913, "kl": 0.58782958984375, "learning_rate": 2.2530209654300403e-06, "loss": 0.0982, "reward": 0.6250000149011612, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.6250000149011612, "rewards/format_reward": 0.0, "step": 599 }, { "clip_ratio": 0.0, "completion_length": 648.7916946411133, "epoch": 0.4, "grad_norm": 0.36228907108306885, "kl": 0.857666015625, "learning_rate": 2.25e-06, "loss": 0.0606, "reward": 0.1666666716337204, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 600 }, { "clip_ratio": 0.0, "completion_length": 605.2500228881836, "epoch": 0.40066666666666667, "grad_norm": 0.7970390319824219, "kl": 1.2943115234375, "learning_rate": 2.246974973006195e-06, "loss": 0.0472, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 601 }, { "clip_ratio": 0.0, "completion_length": 625.2916793823242, "epoch": 0.4013333333333333, "grad_norm": 0.1986081898212433, "kl": 0.5948486328125, "learning_rate": 2.243945900830413e-06, "loss": 0.0292, "reward": 0.375, "reward_std": 0.1369306445121765, "rewards/accuracy_reward": 0.375, "rewards/format_reward": 0.0, "step": 602 }, { "clip_ratio": 0.0, "completion_length": 535.9583511352539, "epoch": 0.402, "grad_norm": 0.5059679746627808, "kl": 1.02880859375, "learning_rate": 2.2409127998763466e-06, "loss": 0.025, "reward": 0.125, "reward_std": 0.1369306445121765, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "step": 603 }, { "clip_ratio": 0.0, "completion_length": 664.6250228881836, "epoch": 0.4026666666666667, "grad_norm": 0.4000336229801178, "kl": 0.623779296875, "learning_rate": 2.237875686569506e-06, "loss": 0.1034, "reward": 0.3333333358168602, "reward_std": 0.4779854342341423, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 604 }, { "clip_ratio": 0.0, "completion_length": 617.8333511352539, "epoch": 0.4033333333333333, "grad_norm": 0.2512640058994293, "kl": 0.42950439453125, "learning_rate": 2.2348345773571304e-06, "loss": 0.1022, "reward": 0.5, "reward_std": 0.273861289024353, "rewards/accuracy_reward": 0.5, "rewards/format_reward": 0.0, "step": 605 }, { "clip_ratio": 0.0, "completion_length": 736.7083435058594, "epoch": 0.404, "grad_norm": 0.305877685546875, "kl": 1.13385009765625, "learning_rate": 2.231789488708099e-06, "loss": 0.064, "reward": 0.3333333432674408, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 606 }, { "clip_ratio": 0.0, "completion_length": 630.5833587646484, "epoch": 0.4046666666666667, "grad_norm": 0.26971396803855896, "kl": 0.48846435546875, "learning_rate": 2.2287404371128404e-06, "loss": -0.0027, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 607 }, { "clip_ratio": 0.0, "completion_length": 509.08334732055664, "epoch": 0.4053333333333333, "grad_norm": 0.31828567385673523, "kl": 0.16943359375, "learning_rate": 2.2256874390832447e-06, "loss": 0.0321, "reward": 0.2500000074505806, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 608 }, { "clip_ratio": 0.0, "completion_length": 770.4166870117188, "epoch": 0.406, "grad_norm": 0.21839874982833862, "kl": 0.4537353515625, "learning_rate": 2.222630511152573e-06, "loss": -0.0071, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 609 }, { "clip_ratio": 0.0, "completion_length": 498.7916793823242, "epoch": 0.4066666666666667, "grad_norm": 0.12451591342687607, "kl": 0.110595703125, "learning_rate": 2.2195696698753695e-06, "loss": 0.0305, "reward": 0.2916666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.2916666679084301, "rewards/format_reward": 0.0, "step": 610 }, { "clip_ratio": 0.0, "completion_length": 397.75000762939453, "epoch": 0.4073333333333333, "grad_norm": 0.2807327210903168, "kl": 0.1668701171875, "learning_rate": 2.21650493182737e-06, "loss": -0.0115, "reward": 0.3750000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.3750000037252903, "rewards/format_reward": 0.0, "step": 611 }, { "clip_ratio": 0.0, "completion_length": 544.6666870117188, "epoch": 0.408, "grad_norm": 0.29039958119392395, "kl": 0.2598876953125, "learning_rate": 2.213436313605413e-06, "loss": 0.0686, "reward": 0.2916666679084301, "reward_std": 0.37592335790395737, "rewards/accuracy_reward": 0.2916666679084301, "rewards/format_reward": 0.0, "step": 612 }, { "clip_ratio": 0.0, "completion_length": 860.0416870117188, "epoch": 0.4086666666666667, "grad_norm": 0.20192325115203857, "kl": 0.55029296875, "learning_rate": 2.21036383182735e-06, "loss": 0.0233, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 613 }, { "clip_ratio": 0.0, "completion_length": 511.25001525878906, "epoch": 0.4093333333333333, "grad_norm": 0.46251243352890015, "kl": 0.204681396484375, "learning_rate": 2.2072875031319556e-06, "loss": 0.1332, "reward": 0.5833333432674408, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.5833333432674408, "rewards/format_reward": 0.0, "step": 614 }, { "clip_ratio": 0.0, "completion_length": 565.5000305175781, "epoch": 0.41, "grad_norm": 0.4453868865966797, "kl": 0.3663330078125, "learning_rate": 2.204207344178836e-06, "loss": 0.0343, "reward": 0.3750000149011612, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.3750000149011612, "rewards/format_reward": 0.0, "step": 615 }, { "clip_ratio": 0.0, "completion_length": 579.8333435058594, "epoch": 0.4106666666666667, "grad_norm": 0.6900433897972107, "kl": 0.46533203125, "learning_rate": 2.2011233716483416e-06, "loss": 0.0949, "reward": 0.4166666828095913, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.4166666828095913, "rewards/format_reward": 0.0, "step": 616 }, { "clip_ratio": 0.0, "completion_length": 705.5000076293945, "epoch": 0.41133333333333333, "grad_norm": 0.6568426489830017, "kl": 0.48712158203125, "learning_rate": 2.1980356022414723e-06, "loss": 0.1248, "reward": 0.3333333469927311, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.3333333469927311, "rewards/format_reward": 0.0, "step": 617 }, { "clip_ratio": 0.0, "completion_length": 557.1666717529297, "epoch": 0.412, "grad_norm": 0.44196656346321106, "kl": 0.47711181640625, "learning_rate": 2.1949440526797927e-06, "loss": 0.0931, "reward": 0.4583333432674408, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.4583333432674408, "rewards/format_reward": 0.0, "step": 618 }, { "clip_ratio": 0.0, "completion_length": 613.6666831970215, "epoch": 0.4126666666666667, "grad_norm": 0.20235884189605713, "kl": 0.4609375, "learning_rate": 2.191848739705336e-06, "loss": 0.0603, "reward": 0.3333333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 619 }, { "clip_ratio": 0.0, "completion_length": 555.2500114440918, "epoch": 0.41333333333333333, "grad_norm": 0.48974883556365967, "kl": 0.4410400390625, "learning_rate": 2.1887496800805174e-06, "loss": 0.1532, "reward": 0.291666679084301, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.291666679084301, "rewards/format_reward": 0.0, "step": 620 }, { "clip_ratio": 0.0, "completion_length": 652.2083511352539, "epoch": 0.414, "grad_norm": 0.25655993819236755, "kl": 0.80621337890625, "learning_rate": 2.1856468905880415e-06, "loss": 0.0328, "reward": 0.25, "reward_std": 0.0, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "step": 621 }, { "clip_ratio": 0.0, "completion_length": 779.0000228881836, "epoch": 0.4146666666666667, "grad_norm": 0.46901392936706543, "kl": 0.9501953125, "learning_rate": 2.1825403880308107e-06, "loss": 0.1634, "reward": 0.2500000074505806, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 622 }, { "clip_ratio": 0.0, "completion_length": 585.1666870117188, "epoch": 0.41533333333333333, "grad_norm": 0.45193102955818176, "kl": 1.19921875, "learning_rate": 2.1794301892318363e-06, "loss": 0.1661, "reward": 0.3333333432674408, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 623 }, { "clip_ratio": 0.0, "completion_length": 850.8750152587891, "epoch": 0.416, "grad_norm": 0.4520170986652374, "kl": 1.623046875, "learning_rate": 2.1763163110341462e-06, "loss": 0.1386, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 624 }, { "clip_ratio": 0.0, "completion_length": 683.1250305175781, "epoch": 0.4166666666666667, "grad_norm": 0.5148462653160095, "kl": 1.3427734375, "learning_rate": 2.1731987703006935e-06, "loss": 0.0557, "reward": 0.2500000111758709, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.2500000111758709, "rewards/format_reward": 0.0, "step": 625 }, { "clip_ratio": 0.0, "completion_length": 511.2500114440918, "epoch": 0.41733333333333333, "grad_norm": 1.080418348312378, "kl": 1.403076171875, "learning_rate": 2.1700775839142652e-06, "loss": 0.2556, "reward": 0.6666666716337204, "reward_std": 0.40296074748039246, "rewards/accuracy_reward": 0.6666666716337204, "rewards/format_reward": 0.0, "step": 626 }, { "clip_ratio": 0.0, "completion_length": 530.9166717529297, "epoch": 0.418, "grad_norm": 0.44552168250083923, "kl": 1.77880859375, "learning_rate": 2.166952768777391e-06, "loss": 0.1845, "reward": 0.5416666716337204, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.5416666716337204, "rewards/format_reward": 0.0, "step": 627 }, { "clip_ratio": 0.0, "completion_length": 576.4583435058594, "epoch": 0.4186666666666667, "grad_norm": 0.3573629558086395, "kl": 1.2578125, "learning_rate": 2.1638243418122534e-06, "loss": 0.0959, "reward": 0.1250000037252903, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 628 }, { "clip_ratio": 0.0, "completion_length": 629.2916793823242, "epoch": 0.41933333333333334, "grad_norm": 0.7354836463928223, "kl": 1.77001953125, "learning_rate": 2.1606923199605926e-06, "loss": 0.0928, "reward": 0.0416666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 629 }, { "clip_ratio": 0.0, "completion_length": 666.9166946411133, "epoch": 0.42, "grad_norm": 0.7089085578918457, "kl": 1.55126953125, "learning_rate": 2.157556720183616e-06, "loss": 0.0731, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 630 }, { "clip_ratio": 0.0, "completion_length": 781.5833740234375, "epoch": 0.4206666666666667, "grad_norm": 1.404771327972412, "kl": 3.70281982421875, "learning_rate": 2.1544175594619103e-06, "loss": 0.1607, "reward": 0.1250000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 631 }, { "clip_ratio": 0.0, "completion_length": 607.5000076293945, "epoch": 0.42133333333333334, "grad_norm": 0.6024755835533142, "kl": 1.1739501953125, "learning_rate": 2.151274854795342e-06, "loss": 0.1454, "reward": 0.4583333507180214, "reward_std": 0.4893604442477226, "rewards/accuracy_reward": 0.4583333507180214, "rewards/format_reward": 0.0, "step": 632 }, { "clip_ratio": 0.0, "completion_length": 808.2917022705078, "epoch": 0.422, "grad_norm": 0.2530776560306549, "kl": 1.2333984375, "learning_rate": 2.1481286232029736e-06, "loss": 0.081, "reward": 0.125, "reward_std": 0.1369306445121765, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "step": 633 }, { "clip_ratio": 0.0, "completion_length": 480.37500762939453, "epoch": 0.4226666666666667, "grad_norm": 1.0031018257141113, "kl": 0.634521484375, "learning_rate": 2.1449788817229644e-06, "loss": 0.1764, "reward": 0.4583333507180214, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.4583333507180214, "rewards/format_reward": 0.0, "step": 634 }, { "clip_ratio": 0.0, "completion_length": 658.3750152587891, "epoch": 0.42333333333333334, "grad_norm": 0.42961838841438293, "kl": 1.02294921875, "learning_rate": 2.1418256474124813e-06, "loss": 0.0575, "reward": 0.125, "reward_std": 0.1369306445121765, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "step": 635 }, { "clip_ratio": 0.0, "completion_length": 736.0833435058594, "epoch": 0.424, "grad_norm": 0.5219307541847229, "kl": 1.18359375, "learning_rate": 2.138668937347609e-06, "loss": 0.0762, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 636 }, { "clip_ratio": 0.0, "completion_length": 540.2916793823242, "epoch": 0.4246666666666667, "grad_norm": 0.6517496705055237, "kl": 1.1383056640625, "learning_rate": 2.135508768623252e-06, "loss": 0.0696, "reward": 0.2916666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.2916666679084301, "rewards/format_reward": 0.0, "step": 637 }, { "clip_ratio": 0.0, "completion_length": 644.0000152587891, "epoch": 0.42533333333333334, "grad_norm": 0.32598933577537537, "kl": 1.47430419921875, "learning_rate": 2.132345158353047e-06, "loss": 0.1878, "reward": 0.3333333358168602, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 638 }, { "clip_ratio": 0.0, "completion_length": 624.2083511352539, "epoch": 0.426, "grad_norm": 0.5203576683998108, "kl": 1.29736328125, "learning_rate": 2.129178123669266e-06, "loss": 0.104, "reward": 0.1666666716337204, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 639 }, { "clip_ratio": 0.0, "completion_length": 521.0833473205566, "epoch": 0.4266666666666667, "grad_norm": 0.3867467939853668, "kl": 0.58905029296875, "learning_rate": 2.126007681722727e-06, "loss": 0.0507, "reward": 0.4166666865348816, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.4166666865348816, "rewards/format_reward": 0.0, "step": 640 }, { "clip_ratio": 0.0, "completion_length": 576.6250152587891, "epoch": 0.42733333333333334, "grad_norm": 0.20481206476688385, "kl": 0.494140625, "learning_rate": 2.1228338496826998e-06, "loss": 0.0226, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 641 }, { "clip_ratio": 0.0, "completion_length": 703.9166870117188, "epoch": 0.428, "grad_norm": 0.16392925381660461, "kl": 0.218505859375, "learning_rate": 2.119656644736813e-06, "loss": 0.0416, "reward": 0.3333333358168602, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 642 }, { "clip_ratio": 0.0, "completion_length": 495.29168701171875, "epoch": 0.42866666666666664, "grad_norm": 1.447789192199707, "kl": 0.70623779296875, "learning_rate": 2.1164760840909595e-06, "loss": 0.0977, "reward": 0.375, "reward_std": 0.1369306445121765, "rewards/accuracy_reward": 0.375, "rewards/format_reward": 0.0, "step": 643 }, { "clip_ratio": 0.0, "completion_length": 593.3333587646484, "epoch": 0.42933333333333334, "grad_norm": 0.606586754322052, "kl": 0.31951904296875, "learning_rate": 2.113292184969207e-06, "loss": 0.0943, "reward": 0.3333333469927311, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.3333333469927311, "rewards/format_reward": 0.0, "step": 644 }, { "clip_ratio": 0.0, "completion_length": 677.2083511352539, "epoch": 0.43, "grad_norm": 0.2526169419288635, "kl": 0.625, "learning_rate": 2.1101049646137005e-06, "loss": 0.0293, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 645 }, { "clip_ratio": 0.0, "completion_length": 566.7083511352539, "epoch": 0.43066666666666664, "grad_norm": 0.38385042548179626, "kl": 0.160400390625, "learning_rate": 2.106914440284572e-06, "loss": 0.1129, "reward": 0.3333333358168602, "reward_std": 0.40296074748039246, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 646 }, { "clip_ratio": 0.0, "completion_length": 683.7916793823242, "epoch": 0.43133333333333335, "grad_norm": 0.4661238491535187, "kl": 0.29547119140625, "learning_rate": 2.1037206292598443e-06, "loss": 0.1357, "reward": 0.2500000074505806, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 647 }, { "clip_ratio": 0.0, "completion_length": 353.8333511352539, "epoch": 0.432, "grad_norm": 0.41052526235580444, "kl": 0.148193359375, "learning_rate": 2.100523548835343e-06, "loss": 0.1191, "reward": 0.5833333432674408, "reward_std": 0.3872983753681183, "rewards/accuracy_reward": 0.5833333432674408, "rewards/format_reward": 0.0, "step": 648 }, { "clip_ratio": 0.0, "completion_length": 478.5416717529297, "epoch": 0.43266666666666664, "grad_norm": 0.335597962141037, "kl": 0.396575927734375, "learning_rate": 2.097323216324593e-06, "loss": 0.0849, "reward": 0.541666679084301, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.541666679084301, "rewards/format_reward": 0.0, "step": 649 }, { "clip_ratio": 0.0, "completion_length": 669.8750228881836, "epoch": 0.43333333333333335, "grad_norm": 0.14245618879795074, "kl": 0.39404296875, "learning_rate": 2.0941196490587354e-06, "loss": 0.0088, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 650 }, { "clip_ratio": 0.0, "completion_length": 645.0000152587891, "epoch": 0.434, "grad_norm": 0.30782151222229004, "kl": 0.7548828125, "learning_rate": 2.090912864386427e-06, "loss": 0.0241, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 651 }, { "clip_ratio": 0.0, "completion_length": 745.5000305175781, "epoch": 0.43466666666666665, "grad_norm": 0.07350896298885345, "kl": 0.21478271484375, "learning_rate": 2.0877028796737477e-06, "loss": 0.0089, "reward": 0.25, "reward_std": 0.0, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "step": 652 }, { "clip_ratio": 0.0, "completion_length": 373.33333587646484, "epoch": 0.43533333333333335, "grad_norm": 0.33463650941848755, "kl": 0.17083740234375, "learning_rate": 2.0844897123041086e-06, "loss": -0.001, "reward": 0.1250000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 653 }, { "clip_ratio": 0.0, "completion_length": 687.5416946411133, "epoch": 0.436, "grad_norm": 0.23459726572036743, "kl": 0.5841064453125, "learning_rate": 2.0812733796781545e-06, "loss": 0.0831, "reward": 0.0833333358168602, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 654 }, { "clip_ratio": 0.0, "completion_length": 431.50002670288086, "epoch": 0.43666666666666665, "grad_norm": 0.33114251494407654, "kl": 0.20587158203125, "learning_rate": 2.0780538992136728e-06, "loss": 0.0101, "reward": 0.541666679084301, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.541666679084301, "rewards/format_reward": 0.0, "step": 655 }, { "clip_ratio": 0.0, "completion_length": 511.0416793823242, "epoch": 0.43733333333333335, "grad_norm": 0.9682897329330444, "kl": 0.4189453125, "learning_rate": 2.0748312883454963e-06, "loss": 0.0749, "reward": 0.6250000260770321, "reward_std": 0.43528568744659424, "rewards/accuracy_reward": 0.6250000260770321, "rewards/format_reward": 0.0, "step": 656 }, { "clip_ratio": 0.0, "completion_length": 620.5833435058594, "epoch": 0.438, "grad_norm": 1.0086687803268433, "kl": 0.818359375, "learning_rate": 2.0716055645254116e-06, "loss": 0.1586, "reward": 0.3333333469927311, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.3333333469927311, "rewards/format_reward": 0.0, "step": 657 }, { "clip_ratio": 0.0, "completion_length": 683.4583435058594, "epoch": 0.43866666666666665, "grad_norm": 0.2298087477684021, "kl": 0.5823974609375, "learning_rate": 2.068376745222062e-06, "loss": 0.0805, "reward": 0.2500000074505806, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 658 }, { "clip_ratio": 0.0, "completion_length": 639.6666870117188, "epoch": 0.43933333333333335, "grad_norm": 1.2123403549194336, "kl": 1.45068359375, "learning_rate": 2.0651448479208558e-06, "loss": 0.2186, "reward": 0.3333333469927311, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.3333333469927311, "rewards/format_reward": 0.0, "step": 659 }, { "clip_ratio": 0.0, "completion_length": 692.2916831970215, "epoch": 0.44, "grad_norm": 1.4683496952056885, "kl": 1.641357421875, "learning_rate": 2.061909890123868e-06, "loss": 0.0967, "reward": 0.3333333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 660 }, { "clip_ratio": 0.0, "completion_length": 481.375, "epoch": 0.44066666666666665, "grad_norm": 3.1196844577789307, "kl": 2.5697021484375, "learning_rate": 2.058671889349749e-06, "loss": 0.283, "reward": 0.4583333507180214, "reward_std": 0.43528567254543304, "rewards/accuracy_reward": 0.4583333507180214, "rewards/format_reward": 0.0, "step": 661 }, { "clip_ratio": 0.0, "completion_length": 474.1666831970215, "epoch": 0.44133333333333336, "grad_norm": 1.5231492519378662, "kl": 1.5350341796875, "learning_rate": 2.055430863133628e-06, "loss": 0.0495, "reward": 0.1666666716337204, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 662 }, { "clip_ratio": 0.0, "completion_length": 610.5833511352539, "epoch": 0.442, "grad_norm": 1.601920247077942, "kl": 1.76800537109375, "learning_rate": 2.0521868290270174e-06, "loss": 0.0809, "reward": 0.3333333432674408, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 663 }, { "clip_ratio": 0.0, "completion_length": 665.2916870117188, "epoch": 0.44266666666666665, "grad_norm": 0.6220554113388062, "kl": 0.962890625, "learning_rate": 2.048939804597718e-06, "loss": 0.0581, "reward": 0.1250000037252903, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 664 }, { "clip_ratio": 0.0, "completion_length": 549.7916870117188, "epoch": 0.44333333333333336, "grad_norm": 1.386452078819275, "kl": 2.646484375, "learning_rate": 2.0456898074297275e-06, "loss": 0.3147, "reward": 0.3750000037252903, "reward_std": 0.5050228163599968, "rewards/accuracy_reward": 0.3750000037252903, "rewards/format_reward": 0.0, "step": 665 }, { "clip_ratio": 0.0, "completion_length": 616.2500152587891, "epoch": 0.444, "grad_norm": 0.9781052470207214, "kl": 2.40966796875, "learning_rate": 2.0424368551231384e-06, "loss": 0.1134, "reward": 0.2083333358168602, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.2083333358168602, "rewards/format_reward": 0.0, "step": 666 }, { "clip_ratio": 0.0, "completion_length": 399.9583396911621, "epoch": 0.44466666666666665, "grad_norm": 0.26444491744041443, "kl": 0.66796875, "learning_rate": 2.03918096529405e-06, "loss": 0.1401, "reward": 0.4166666679084301, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.4166666679084301, "rewards/format_reward": 0.0, "step": 667 }, { "clip_ratio": 0.0, "completion_length": 798.9583435058594, "epoch": 0.44533333333333336, "grad_norm": 0.4013528525829315, "kl": 1.3095703125, "learning_rate": 2.035922155574466e-06, "loss": 0.1169, "reward": 0.1666666716337204, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 668 }, { "clip_ratio": 0.0, "completion_length": 748.4166717529297, "epoch": 0.446, "grad_norm": 0.5632343292236328, "kl": 1.8125, "learning_rate": 2.0326604436122056e-06, "loss": 0.0555, "reward": 0.125, "reward_std": 0.1369306445121765, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "step": 669 }, { "clip_ratio": 0.0, "completion_length": 654.9583587646484, "epoch": 0.44666666666666666, "grad_norm": 0.6248593926429749, "kl": 1.578125, "learning_rate": 2.0293958470708033e-06, "loss": 0.1075, "reward": 0.2500000074505806, "reward_std": 0.3872983753681183, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 670 }, { "clip_ratio": 0.0, "completion_length": 555.6250152587891, "epoch": 0.44733333333333336, "grad_norm": 0.34907266497612, "kl": 0.541015625, "learning_rate": 2.0261283836294155e-06, "loss": 0.0304, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 671 }, { "clip_ratio": 0.0, "completion_length": 652.5833435058594, "epoch": 0.448, "grad_norm": 1.5893522500991821, "kl": 1.5380859375, "learning_rate": 2.022858070982723e-06, "loss": 0.1218, "reward": 0.2500000074505806, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 672 }, { "clip_ratio": 0.0, "completion_length": 597.2916870117188, "epoch": 0.44866666666666666, "grad_norm": 1.04214346408844, "kl": 0.88330078125, "learning_rate": 2.019584926840837e-06, "loss": 0.1743, "reward": 0.2500000111758709, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.2500000111758709, "rewards/format_reward": 0.0, "step": 673 }, { "clip_ratio": 0.0, "completion_length": 544.125, "epoch": 0.4493333333333333, "grad_norm": 0.5386791229248047, "kl": 0.8076171875, "learning_rate": 2.016308968929203e-06, "loss": 0.2068, "reward": 0.1666666679084301, "reward_std": 0.23899272084236145, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 674 }, { "clip_ratio": 0.0, "completion_length": 676.4583435058594, "epoch": 0.45, "grad_norm": 0.24457289278507233, "kl": 0.456146240234375, "learning_rate": 2.0130302149885033e-06, "loss": 0.0045, "reward": 0.2083333432674408, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.2083333432674408, "rewards/format_reward": 0.0, "step": 675 }, { "clip_ratio": 0.0, "completion_length": 604.3333587646484, "epoch": 0.45066666666666666, "grad_norm": 0.5820838809013367, "kl": 0.70916748046875, "learning_rate": 2.0097486827745623e-06, "loss": 0.0989, "reward": 0.1666666679084301, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 676 }, { "clip_ratio": 0.0, "completion_length": 549.7500152587891, "epoch": 0.4513333333333333, "grad_norm": 0.42628082633018494, "kl": 0.3828125, "learning_rate": 2.0064643900582487e-06, "loss": 0.0228, "reward": 0.2500000111758709, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.2500000111758709, "rewards/format_reward": 0.0, "step": 677 }, { "clip_ratio": 0.0, "completion_length": 781.9166870117188, "epoch": 0.452, "grad_norm": 1.1463125944137573, "kl": 1.03515625, "learning_rate": 2.0031773546253826e-06, "loss": 0.1573, "reward": 0.3333333395421505, "reward_std": 0.4971916303038597, "rewards/accuracy_reward": 0.3333333395421505, "rewards/format_reward": 0.0, "step": 678 }, { "clip_ratio": 0.0, "completion_length": 617.7083435058594, "epoch": 0.45266666666666666, "grad_norm": 1.0379836559295654, "kl": 1.253173828125, "learning_rate": 1.9998875942766365e-06, "loss": 0.2065, "reward": 0.3750000111758709, "reward_std": 0.4893604516983032, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 679 }, { "clip_ratio": 0.0, "completion_length": 720.3750305175781, "epoch": 0.4533333333333333, "grad_norm": 0.5733534097671509, "kl": 1.640625, "learning_rate": 1.9965951268274372e-06, "loss": 0.1782, "reward": 0.1666666716337204, "reward_std": 0.40824828296899796, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 680 }, { "clip_ratio": 0.0, "completion_length": 713.5000228881836, "epoch": 0.454, "grad_norm": 1.3562406301498413, "kl": 2.5869140625, "learning_rate": 1.993299970107875e-06, "loss": 0.2302, "reward": 0.2916666753590107, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.2916666753590107, "rewards/format_reward": 0.0, "step": 681 }, { "clip_ratio": 0.0, "completion_length": 515.3333511352539, "epoch": 0.45466666666666666, "grad_norm": 1.4147708415985107, "kl": 2.197998046875, "learning_rate": 1.9900021419626017e-06, "loss": 0.1679, "reward": 0.5833333432674408, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.5833333432674408, "rewards/format_reward": 0.0, "step": 682 }, { "clip_ratio": 0.0, "completion_length": 452.54167556762695, "epoch": 0.4553333333333333, "grad_norm": 1.0796515941619873, "kl": 1.50921630859375, "learning_rate": 1.986701660250736e-06, "loss": 0.2223, "reward": 0.5416666753590107, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.5416666753590107, "rewards/format_reward": 0.0, "step": 683 }, { "clip_ratio": 0.0, "completion_length": 696.5000305175781, "epoch": 0.456, "grad_norm": 8.45889663696289, "kl": 4.1953125, "learning_rate": 1.983398542845767e-06, "loss": 0.2175, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 684 }, { "clip_ratio": 0.0, "completion_length": 619.6666717529297, "epoch": 0.45666666666666667, "grad_norm": 2.7459683418273926, "kl": 4.1767578125, "learning_rate": 1.980092807635457e-06, "loss": 0.4043, "reward": 0.3750000074505806, "reward_std": 0.39512956142425537, "rewards/accuracy_reward": 0.3750000074505806, "rewards/format_reward": 0.0, "step": 685 }, { "clip_ratio": 0.0, "completion_length": 341.4583435058594, "epoch": 0.4573333333333333, "grad_norm": 2.161928415298462, "kl": 1.47528076171875, "learning_rate": 1.976784472521747e-06, "loss": 0.3198, "reward": 0.6666666977107525, "reward_std": 0.40824829041957855, "rewards/accuracy_reward": 0.6666666977107525, "rewards/format_reward": 0.0, "step": 686 }, { "clip_ratio": 0.0, "completion_length": 585.9583587646484, "epoch": 0.458, "grad_norm": 0.7706136107444763, "kl": 1.2431640625, "learning_rate": 1.973473555420654e-06, "loss": 0.1154, "reward": 0.2500000037252903, "reward_std": 0.4431168735027313, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.0, "step": 687 }, { "clip_ratio": 0.0, "completion_length": 762.7916793823242, "epoch": 0.45866666666666667, "grad_norm": 2.8532841205596924, "kl": 4.845703125, "learning_rate": 1.9701600742621796e-06, "loss": 0.2783, "reward": 0.1666666679084301, "reward_std": 0.23899272084236145, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 688 }, { "clip_ratio": 0.0, "completion_length": 522.5000305175781, "epoch": 0.4593333333333333, "grad_norm": 2.187533140182495, "kl": 2.3092041015625, "learning_rate": 1.96684404699021e-06, "loss": 0.0896, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 689 }, { "clip_ratio": 0.0, "completion_length": 651.5000152587891, "epoch": 0.46, "grad_norm": 1.916913628578186, "kl": 1.028778076171875, "learning_rate": 1.963525491562421e-06, "loss": 0.2051, "reward": 0.4583333507180214, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.4583333507180214, "rewards/format_reward": 0.0, "step": 690 }, { "clip_ratio": 0.0, "completion_length": 655.1666793823242, "epoch": 0.46066666666666667, "grad_norm": 0.6601982116699219, "kl": 3.0712890625, "learning_rate": 1.960204425950179e-06, "loss": 0.2893, "reward": 0.5000000037252903, "reward_std": 0.3680921792984009, "rewards/accuracy_reward": 0.5000000037252903, "rewards/format_reward": 0.0, "step": 691 }, { "clip_ratio": 0.0, "completion_length": 478.5833549499512, "epoch": 0.4613333333333333, "grad_norm": 0.5131397843360901, "kl": 1.560791015625, "learning_rate": 1.9568808681384415e-06, "loss": 0.1069, "reward": 0.3333333469927311, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.3333333469927311, "rewards/format_reward": 0.0, "step": 692 }, { "clip_ratio": 0.0, "completion_length": 836.5833587646484, "epoch": 0.462, "grad_norm": 0.9719308018684387, "kl": 2.3818359375, "learning_rate": 1.953554836125667e-06, "loss": 0.1353, "reward": 0.1250000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 693 }, { "clip_ratio": 0.0, "completion_length": 547.5833435058594, "epoch": 0.46266666666666667, "grad_norm": 0.7397841215133667, "kl": 1.77105712890625, "learning_rate": 1.9502263479237084e-06, "loss": 0.2917, "reward": 0.3750000111758709, "reward_std": 0.3602609932422638, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 694 }, { "clip_ratio": 0.0, "completion_length": 587.5833473205566, "epoch": 0.4633333333333333, "grad_norm": 0.26176998019218445, "kl": 0.2764892578125, "learning_rate": 1.9468954215577227e-06, "loss": 0.0431, "reward": 0.4583333395421505, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.4583333395421505, "rewards/format_reward": 0.0, "step": 695 }, { "clip_ratio": 0.0, "completion_length": 393.04168701171875, "epoch": 0.464, "grad_norm": 1.957637906074524, "kl": 0.993377685546875, "learning_rate": 1.9435620750660703e-06, "loss": 0.2564, "reward": 0.6250000149011612, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.6250000149011612, "rewards/format_reward": 0.0, "step": 696 }, { "clip_ratio": 0.0, "completion_length": 536.1666870117188, "epoch": 0.4646666666666667, "grad_norm": 0.43475911021232605, "kl": 1.377685546875, "learning_rate": 1.940226326500216e-06, "loss": 0.0679, "reward": 0.4166666679084301, "reward_std": 0.23899272084236145, "rewards/accuracy_reward": 0.4166666679084301, "rewards/format_reward": 0.0, "step": 697 }, { "clip_ratio": 0.0, "completion_length": 633.2917022705078, "epoch": 0.4653333333333333, "grad_norm": 2.954249382019043, "kl": 2.52777099609375, "learning_rate": 1.9368881939246333e-06, "loss": 0.1182, "reward": 0.2916666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.2916666679084301, "rewards/format_reward": 0.0, "step": 698 }, { "clip_ratio": 0.0, "completion_length": 541.9166946411133, "epoch": 0.466, "grad_norm": 0.7058267593383789, "kl": 1.12188720703125, "learning_rate": 1.9335476954167073e-06, "loss": 0.0404, "reward": 0.2916666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.2916666679084301, "rewards/format_reward": 0.0, "step": 699 }, { "clip_ratio": 0.0, "completion_length": 630.7916793823242, "epoch": 0.4666666666666667, "grad_norm": 0.7592703104019165, "kl": 1.87451171875, "learning_rate": 1.9302048490666355e-06, "loss": 0.2585, "reward": 0.3333333395421505, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.3333333395421505, "rewards/format_reward": 0.0, "step": 700 }, { "clip_ratio": 0.0, "completion_length": 565.9166870117188, "epoch": 0.4673333333333333, "grad_norm": 0.368221253156662, "kl": 0.52325439453125, "learning_rate": 1.9268596729773286e-06, "loss": 0.024, "reward": 0.2916666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.2916666679084301, "rewards/format_reward": 0.0, "step": 701 }, { "clip_ratio": 0.0, "completion_length": 550.0416717529297, "epoch": 0.468, "grad_norm": 0.5162802338600159, "kl": 0.955078125, "learning_rate": 1.923512185264315e-06, "loss": 0.0363, "reward": 0.1250000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 702 }, { "clip_ratio": 0.0, "completion_length": 572.5833511352539, "epoch": 0.4686666666666667, "grad_norm": 0.4348187744617462, "kl": 0.6351318359375, "learning_rate": 1.9201624040556406e-06, "loss": 0.0907, "reward": 0.2083333432674408, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.2083333432674408, "rewards/format_reward": 0.0, "step": 703 }, { "clip_ratio": 0.0, "completion_length": 571.2916870117188, "epoch": 0.4693333333333333, "grad_norm": 0.3552999496459961, "kl": 1.068359375, "learning_rate": 1.916810347491772e-06, "loss": 0.0558, "reward": 0.125, "reward_std": 0.1369306445121765, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "step": 704 }, { "clip_ratio": 0.0, "completion_length": 646.2083587646484, "epoch": 0.47, "grad_norm": 0.28826063871383667, "kl": 0.38397216796875, "learning_rate": 1.9134560337254986e-06, "loss": 0.009, "reward": 0.1666666716337204, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 705 }, { "clip_ratio": 0.0, "completion_length": 555.1666870117188, "epoch": 0.4706666666666667, "grad_norm": 1.1848900318145752, "kl": 0.669921875, "learning_rate": 1.9100994809218323e-06, "loss": 0.1271, "reward": 0.2500000111758709, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.2500000111758709, "rewards/format_reward": 0.0, "step": 706 }, { "clip_ratio": 0.0, "completion_length": 606.0000152587891, "epoch": 0.4713333333333333, "grad_norm": 0.351146936416626, "kl": 0.21124267578125, "learning_rate": 1.906740707257911e-06, "loss": 0.0014, "reward": 0.0833333358168602, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 707 }, { "clip_ratio": 0.0, "completion_length": 474.8750228881836, "epoch": 0.472, "grad_norm": 0.5447246432304382, "kl": 0.4820556640625, "learning_rate": 1.9033797309228985e-06, "loss": 0.0832, "reward": 0.2083333432674408, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.2083333432674408, "rewards/format_reward": 0.0, "step": 708 }, { "clip_ratio": 0.0, "completion_length": 637.8750152587891, "epoch": 0.4726666666666667, "grad_norm": 3.4836885929107666, "kl": 1.76318359375, "learning_rate": 1.9000165701178883e-06, "loss": 0.0913, "reward": 0.0833333358168602, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 709 }, { "clip_ratio": 0.0, "completion_length": 549.4166793823242, "epoch": 0.47333333333333333, "grad_norm": 1.6220284700393677, "kl": 0.8037109375, "learning_rate": 1.8966512430558036e-06, "loss": 0.2126, "reward": 0.3333333432674408, "reward_std": 0.23899272084236145, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 710 }, { "clip_ratio": 0.0, "completion_length": 592.8333358764648, "epoch": 0.474, "grad_norm": 0.660271942615509, "kl": 0.93450927734375, "learning_rate": 1.8932837679612973e-06, "loss": 0.1656, "reward": 0.2916666716337204, "reward_std": 0.4701542407274246, "rewards/accuracy_reward": 0.2916666716337204, "rewards/format_reward": 0.0, "step": 711 }, { "clip_ratio": 0.0, "completion_length": 610.3750228881836, "epoch": 0.4746666666666667, "grad_norm": 0.9388156533241272, "kl": 0.580078125, "learning_rate": 1.8899141630706564e-06, "loss": 0.157, "reward": 0.3750000149011612, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.3750000149011612, "rewards/format_reward": 0.0, "step": 712 }, { "clip_ratio": 0.0, "completion_length": 680.3750152587891, "epoch": 0.47533333333333333, "grad_norm": 0.68855220079422, "kl": 1.2584228515625, "learning_rate": 1.886542446631701e-06, "loss": 0.1754, "reward": 0.3333333469927311, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.3333333469927311, "rewards/format_reward": 0.0, "step": 713 }, { "clip_ratio": 0.0, "completion_length": 665.4166870117188, "epoch": 0.476, "grad_norm": 1.0510975122451782, "kl": 0.9696044921875, "learning_rate": 1.8831686369036859e-06, "loss": 0.0916, "reward": 0.2500000111758709, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.2500000111758709, "rewards/format_reward": 0.0, "step": 714 }, { "clip_ratio": 0.0, "completion_length": 436.2916717529297, "epoch": 0.4766666666666667, "grad_norm": 0.5948176980018616, "kl": 0.699462890625, "learning_rate": 1.879792752157203e-06, "loss": 0.0322, "reward": 0.3333333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 715 }, { "clip_ratio": 0.0, "completion_length": 508.8750114440918, "epoch": 0.47733333333333333, "grad_norm": 0.5294050574302673, "kl": 0.629638671875, "learning_rate": 1.876414810674079e-06, "loss": 0.0533, "reward": 0.0833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 716 }, { "clip_ratio": 0.0, "completion_length": 596.7083435058594, "epoch": 0.478, "grad_norm": 0.4353724718093872, "kl": 0.55908203125, "learning_rate": 1.8730348307472826e-06, "loss": 0.1017, "reward": 0.3750000074505806, "reward_std": 0.39512956142425537, "rewards/accuracy_reward": 0.3750000074505806, "rewards/format_reward": 0.0, "step": 717 }, { "clip_ratio": 0.0, "completion_length": 694.2083435058594, "epoch": 0.4786666666666667, "grad_norm": 1.1658200025558472, "kl": 1.41796875, "learning_rate": 1.8696528306808168e-06, "loss": 0.0676, "reward": 0.1666666679084301, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 718 }, { "clip_ratio": 0.0, "completion_length": 586.208366394043, "epoch": 0.47933333333333333, "grad_norm": 0.40458959341049194, "kl": 0.163848876953125, "learning_rate": 1.866268828789628e-06, "loss": 0.1994, "reward": 0.4583333432674408, "reward_std": 0.37592335790395737, "rewards/accuracy_reward": 0.4583333432674408, "rewards/format_reward": 0.0, "step": 719 }, { "clip_ratio": 0.0, "completion_length": 534.5000076293945, "epoch": 0.48, "grad_norm": 0.2218366414308548, "kl": 0.17388916015625, "learning_rate": 1.8628828433995015e-06, "loss": -0.0369, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 720 }, { "clip_ratio": 0.0, "completion_length": 456.00001525878906, "epoch": 0.4806666666666667, "grad_norm": 0.1450730413198471, "kl": 0.31109619140625, "learning_rate": 1.8594948928469657e-06, "loss": 0.0613, "reward": 0.2916666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.2916666679084301, "rewards/format_reward": 0.0, "step": 721 }, { "clip_ratio": 0.0, "completion_length": 661.5833435058594, "epoch": 0.48133333333333334, "grad_norm": 0.3929297626018524, "kl": 0.78955078125, "learning_rate": 1.8561049954791895e-06, "loss": 0.1729, "reward": 0.2916666679084301, "reward_std": 0.37592335790395737, "rewards/accuracy_reward": 0.2916666679084301, "rewards/format_reward": 0.0, "step": 722 }, { "clip_ratio": 0.0, "completion_length": 623.4583435058594, "epoch": 0.482, "grad_norm": 1.276231050491333, "kl": 0.69189453125, "learning_rate": 1.852713169653885e-06, "loss": 0.1778, "reward": 0.3333333432674408, "reward_std": 0.3872983753681183, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 723 }, { "clip_ratio": 0.0, "completion_length": 611.2916870117188, "epoch": 0.4826666666666667, "grad_norm": 0.6580324769020081, "kl": 0.578125, "learning_rate": 1.8493194337392087e-06, "loss": 0.19, "reward": 0.2500000074505806, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 724 }, { "clip_ratio": 0.0, "completion_length": 556.6250114440918, "epoch": 0.48333333333333334, "grad_norm": 0.9195569753646851, "kl": 0.97119140625, "learning_rate": 1.8459238061136603e-06, "loss": 0.0722, "reward": 0.375, "reward_std": 0.1369306445121765, "rewards/accuracy_reward": 0.375, "rewards/format_reward": 0.0, "step": 725 }, { "clip_ratio": 0.0, "completion_length": 551.4166793823242, "epoch": 0.484, "grad_norm": 0.76514732837677, "kl": 0.81201171875, "learning_rate": 1.8425263051659837e-06, "loss": 0.1247, "reward": 0.3333333395421505, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.3333333395421505, "rewards/format_reward": 0.0, "step": 726 }, { "clip_ratio": 0.0, "completion_length": 604.6666870117188, "epoch": 0.4846666666666667, "grad_norm": 0.3653363883495331, "kl": 0.491943359375, "learning_rate": 1.8391269492950679e-06, "loss": 0.121, "reward": 0.2500000111758709, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.2500000111758709, "rewards/format_reward": 0.0, "step": 727 }, { "clip_ratio": 0.0, "completion_length": 552.4166870117188, "epoch": 0.48533333333333334, "grad_norm": 0.6313820481300354, "kl": 0.61676025390625, "learning_rate": 1.8357257569098473e-06, "loss": 0.0335, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 728 }, { "clip_ratio": 0.0, "completion_length": 680.1666717529297, "epoch": 0.486, "grad_norm": 0.3155249357223511, "kl": 0.97882080078125, "learning_rate": 1.8323227464292013e-06, "loss": -0.048, "reward": 0.2083333395421505, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 729 }, { "clip_ratio": 0.0, "completion_length": 460.5833435058594, "epoch": 0.4866666666666667, "grad_norm": 0.6767831444740295, "kl": 0.670654296875, "learning_rate": 1.828917936281855e-06, "loss": -0.0032, "reward": 0.0833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 730 }, { "clip_ratio": 0.0, "completion_length": 554.2916870117188, "epoch": 0.48733333333333334, "grad_norm": 0.34296244382858276, "kl": 0.48919677734375, "learning_rate": 1.8255113449062806e-06, "loss": 0.0212, "reward": 0.25, "reward_std": 0.0, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "step": 731 }, { "clip_ratio": 0.0, "completion_length": 521.1250076293945, "epoch": 0.488, "grad_norm": 0.2963123321533203, "kl": 0.647491455078125, "learning_rate": 1.822102990750595e-06, "loss": 0.0889, "reward": 0.666666679084301, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.666666679084301, "rewards/format_reward": 0.0, "step": 732 }, { "clip_ratio": 0.0, "completion_length": 586.6666793823242, "epoch": 0.4886666666666667, "grad_norm": 0.4581669867038727, "kl": 0.795166015625, "learning_rate": 1.8186928922724618e-06, "loss": 0.0223, "reward": 0.4583333358168602, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.4583333358168602, "rewards/format_reward": 0.0, "step": 733 }, { "clip_ratio": 0.0, "completion_length": 638.0000228881836, "epoch": 0.48933333333333334, "grad_norm": 1.2634106874465942, "kl": 1.48828125, "learning_rate": 1.8152810679389911e-06, "loss": 0.0679, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 734 }, { "clip_ratio": 0.0, "completion_length": 383.54167556762695, "epoch": 0.49, "grad_norm": 0.3851398527622223, "kl": 0.15167236328125, "learning_rate": 1.8118675362266389e-06, "loss": 0.0072, "reward": 0.6666666716337204, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.6666666716337204, "rewards/format_reward": 0.0, "step": 735 }, { "clip_ratio": 0.0, "completion_length": 551.0416870117188, "epoch": 0.49066666666666664, "grad_norm": 2.121330738067627, "kl": 1.2813720703125, "learning_rate": 1.808452315621108e-06, "loss": 0.1873, "reward": 0.375, "reward_std": 0.41079193353652954, "rewards/accuracy_reward": 0.375, "rewards/format_reward": 0.0, "step": 736 }, { "clip_ratio": 0.0, "completion_length": 540.1250076293945, "epoch": 0.49133333333333334, "grad_norm": 0.13291779160499573, "kl": 0.2359619140625, "learning_rate": 1.8050354246172457e-06, "loss": 0.0063, "reward": 0.0416666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 737 }, { "clip_ratio": 0.0, "completion_length": 521.6666793823242, "epoch": 0.492, "grad_norm": 0.6999210119247437, "kl": 0.39068603515625, "learning_rate": 1.8016168817189471e-06, "loss": 0.0113, "reward": 0.3333333432674408, "reward_std": 0.46232306212186813, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 738 }, { "clip_ratio": 0.0, "completion_length": 489.75001525878906, "epoch": 0.49266666666666664, "grad_norm": 1.18009352684021, "kl": 0.58502197265625, "learning_rate": 1.7981967054390517e-06, "loss": 0.1306, "reward": 0.6250000223517418, "reward_std": 0.4701542407274246, "rewards/accuracy_reward": 0.6250000223517418, "rewards/format_reward": 0.0, "step": 739 }, { "clip_ratio": 0.0, "completion_length": 549.5000152587891, "epoch": 0.49333333333333335, "grad_norm": 0.4576450288295746, "kl": 0.921875, "learning_rate": 1.7947749142992453e-06, "loss": 0.0291, "reward": 0.0833333358168602, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 740 }, { "clip_ratio": 0.0, "completion_length": 564.4583587646484, "epoch": 0.494, "grad_norm": 0.23962680995464325, "kl": 0.44024658203125, "learning_rate": 1.791351526829958e-06, "loss": 0.0469, "reward": 0.1666666716337204, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 741 }, { "clip_ratio": 0.0, "completion_length": 557.0000228881836, "epoch": 0.49466666666666664, "grad_norm": 1.0973050594329834, "kl": 0.8212890625, "learning_rate": 1.7879265615702653e-06, "loss": 0.1558, "reward": 0.3333333469927311, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.3333333469927311, "rewards/format_reward": 0.0, "step": 742 }, { "clip_ratio": 0.0, "completion_length": 491.6666831970215, "epoch": 0.49533333333333335, "grad_norm": 1.3176403045654297, "kl": 0.97509765625, "learning_rate": 1.7845000370677873e-06, "loss": 0.1132, "reward": 0.4583333469927311, "reward_std": 0.4701542407274246, "rewards/accuracy_reward": 0.4583333469927311, "rewards/format_reward": 0.0, "step": 743 }, { "clip_ratio": 0.0, "completion_length": 581.9166870117188, "epoch": 0.496, "grad_norm": 1.1696295738220215, "kl": 0.85009765625, "learning_rate": 1.7810719718785873e-06, "loss": 0.2339, "reward": 0.2916666716337204, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.2916666716337204, "rewards/format_reward": 0.0, "step": 744 }, { "clip_ratio": 0.0, "completion_length": 488.70835876464844, "epoch": 0.49666666666666665, "grad_norm": 1.0153611898422241, "kl": 1.1494140625, "learning_rate": 1.7776423845670718e-06, "loss": 0.1353, "reward": 0.4583333507180214, "reward_std": 0.3602609932422638, "rewards/accuracy_reward": 0.4583333507180214, "rewards/format_reward": 0.0, "step": 745 }, { "clip_ratio": 0.0, "completion_length": 801.6667022705078, "epoch": 0.49733333333333335, "grad_norm": 2.896242618560791, "kl": 3.25, "learning_rate": 1.7742112937058924e-06, "loss": 0.225, "reward": 0.0833333358168602, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 746 }, { "clip_ratio": 0.0, "completion_length": 588.7916793823242, "epoch": 0.498, "grad_norm": 0.6042556762695312, "kl": 0.983154296875, "learning_rate": 1.7707787178758402e-06, "loss": 0.0909, "reward": 0.291666679084301, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.291666679084301, "rewards/format_reward": 0.0, "step": 747 }, { "clip_ratio": 0.0, "completion_length": 654.9583435058594, "epoch": 0.49866666666666665, "grad_norm": 4.989830017089844, "kl": 3.44091796875, "learning_rate": 1.76734467566575e-06, "loss": 0.2615, "reward": 0.2500000074505806, "reward_std": 0.3332236111164093, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 748 }, { "clip_ratio": 0.0, "completion_length": 481.7083435058594, "epoch": 0.49933333333333335, "grad_norm": 3.5128233432769775, "kl": 2.3505859375, "learning_rate": 1.7639091856723963e-06, "loss": 0.1783, "reward": 0.3333333432674408, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 749 }, { "clip_ratio": 0.0, "completion_length": 652.6250152587891, "epoch": 0.5, "grad_norm": 1.4844977855682373, "kl": 1.218994140625, "learning_rate": 1.7604722665003958e-06, "loss": 0.0778, "reward": 0.1250000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 750 }, { "clip_ratio": 0.0, "completion_length": 673.4166793823242, "epoch": 0.5006666666666667, "grad_norm": 2.8641974925994873, "kl": 2.951171875, "learning_rate": 1.7570339367621028e-06, "loss": 0.252, "reward": 0.2916666753590107, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.2916666753590107, "rewards/format_reward": 0.0, "step": 751 }, { "clip_ratio": 0.0, "completion_length": 670.9166793823242, "epoch": 0.5013333333333333, "grad_norm": 1.3860492706298828, "kl": 1.377685546875, "learning_rate": 1.753594215077511e-06, "loss": 0.0687, "reward": 0.0833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 752 }, { "clip_ratio": 0.0, "completion_length": 559.8750152587891, "epoch": 0.502, "grad_norm": 0.7746919989585876, "kl": 2.0389404296875, "learning_rate": 1.7501531200741535e-06, "loss": 0.1724, "reward": 0.1666666679084301, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 753 }, { "clip_ratio": 0.0, "completion_length": 611.3750305175781, "epoch": 0.5026666666666667, "grad_norm": 0.5063151717185974, "kl": 0.85888671875, "learning_rate": 1.7467106703869982e-06, "loss": 0.1239, "reward": 0.2916666753590107, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.2916666753590107, "rewards/format_reward": 0.0, "step": 754 }, { "clip_ratio": 0.0, "completion_length": 557.3750152587891, "epoch": 0.5033333333333333, "grad_norm": 0.22199425101280212, "kl": 0.495361328125, "learning_rate": 1.7432668846583513e-06, "loss": 0.0158, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 755 }, { "clip_ratio": 0.0, "completion_length": 384.3333511352539, "epoch": 0.504, "grad_norm": 0.2429209053516388, "kl": 0.2596435546875, "learning_rate": 1.7398217815377524e-06, "loss": 0.0185, "reward": 0.541666679084301, "reward_std": 0.306186206638813, "rewards/accuracy_reward": 0.541666679084301, "rewards/format_reward": 0.0, "step": 756 }, { "clip_ratio": 0.0, "completion_length": 493.7916717529297, "epoch": 0.5046666666666667, "grad_norm": 0.6877415180206299, "kl": 0.4425048828125, "learning_rate": 1.7363753796818765e-06, "loss": 0.0307, "reward": 0.2083333395421505, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 757 }, { "clip_ratio": 0.0, "completion_length": 431.0833435058594, "epoch": 0.5053333333333333, "grad_norm": 1.1375885009765625, "kl": 0.243408203125, "learning_rate": 1.7329276977544314e-06, "loss": 0.0139, "reward": 0.5833333507180214, "reward_std": 0.5163978338241577, "rewards/accuracy_reward": 0.5833333507180214, "rewards/format_reward": 0.0, "step": 758 }, { "clip_ratio": 0.0, "completion_length": 406.0833396911621, "epoch": 0.506, "grad_norm": 0.2464095503091812, "kl": 0.1380615234375, "learning_rate": 1.7294787544260573e-06, "loss": 0.0206, "reward": 0.7083333395421505, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.7083333395421505, "rewards/format_reward": 0.0, "step": 759 }, { "clip_ratio": 0.0, "completion_length": 433.2083435058594, "epoch": 0.5066666666666667, "grad_norm": 0.337140291929245, "kl": 0.3712158203125, "learning_rate": 1.7260285683742248e-06, "loss": 0.0356, "reward": 0.3750000149011612, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.3750000149011612, "rewards/format_reward": 0.0, "step": 760 }, { "clip_ratio": 0.0, "completion_length": 376.4583435058594, "epoch": 0.5073333333333333, "grad_norm": 0.18975117802619934, "kl": 0.056640625, "learning_rate": 1.722577158283135e-06, "loss": 0.0162, "reward": 0.6666666716337204, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.6666666716337204, "rewards/format_reward": 0.0, "step": 761 }, { "clip_ratio": 0.0, "completion_length": 364.3333435058594, "epoch": 0.508, "grad_norm": 0.6894664764404297, "kl": 0.267822265625, "learning_rate": 1.7191245428436176e-06, "loss": -0.0012, "reward": 0.3750000149011612, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.3750000149011612, "rewards/format_reward": 0.0, "step": 762 }, { "clip_ratio": 0.0, "completion_length": 389.5416717529297, "epoch": 0.5086666666666667, "grad_norm": 0.09507434815168381, "kl": 0.04742431640625, "learning_rate": 1.7156707407530293e-06, "loss": -0.0021, "reward": 0.6666666716337204, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.6666666716337204, "rewards/format_reward": 0.0, "step": 763 }, { "clip_ratio": 0.0, "completion_length": 699.9583435058594, "epoch": 0.5093333333333333, "grad_norm": 0.1483691781759262, "kl": 0.3138427734375, "learning_rate": 1.7122157707151534e-06, "loss": 0.0445, "reward": 0.0833333358168602, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 764 }, { "clip_ratio": 0.0, "completion_length": 539.4583511352539, "epoch": 0.51, "grad_norm": 0.4663875699043274, "kl": 0.530517578125, "learning_rate": 1.7087596514400981e-06, "loss": 0.0791, "reward": 0.4583333358168602, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.4583333358168602, "rewards/format_reward": 0.0, "step": 765 }, { "clip_ratio": 0.0, "completion_length": 358.37500762939453, "epoch": 0.5106666666666667, "grad_norm": 0.24930959939956665, "kl": 0.1312255859375, "learning_rate": 1.7053024016441956e-06, "loss": 0.0382, "reward": 0.3333333358168602, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 766 }, { "clip_ratio": 0.0, "completion_length": 509.00001525878906, "epoch": 0.5113333333333333, "grad_norm": 0.3710845410823822, "kl": 0.5489501953125, "learning_rate": 1.7018440400498994e-06, "loss": 0.0324, "reward": 0.2500000037252903, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.0, "step": 767 }, { "clip_ratio": 0.0, "completion_length": 500.1666793823242, "epoch": 0.512, "grad_norm": 0.47989723086357117, "kl": 0.33984375, "learning_rate": 1.698384585385684e-06, "loss": 0.0888, "reward": 0.4166666865348816, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.4166666865348816, "rewards/format_reward": 0.0, "step": 768 }, { "clip_ratio": 0.0, "completion_length": 634.2083587646484, "epoch": 0.5126666666666667, "grad_norm": 1.0814520120620728, "kl": 0.716552734375, "learning_rate": 1.694924056385944e-06, "loss": 0.3144, "reward": 0.2916666716337204, "reward_std": 0.4701542481780052, "rewards/accuracy_reward": 0.2916666716337204, "rewards/format_reward": 0.0, "step": 769 }, { "clip_ratio": 0.0, "completion_length": 507.7916946411133, "epoch": 0.5133333333333333, "grad_norm": 0.26178884506225586, "kl": 0.23455810546875, "learning_rate": 1.6914624717908924e-06, "loss": 0.0233, "reward": 0.25, "reward_std": 0.273861289024353, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "step": 770 }, { "clip_ratio": 0.0, "completion_length": 478.75001525878906, "epoch": 0.514, "grad_norm": 0.4773680865764618, "kl": 0.66754150390625, "learning_rate": 1.6879998503464564e-06, "loss": 0.0684, "reward": 0.4166666679084301, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.4166666679084301, "rewards/format_reward": 0.0, "step": 771 }, { "clip_ratio": 0.0, "completion_length": 453.0416793823242, "epoch": 0.5146666666666667, "grad_norm": 0.7859503030776978, "kl": 1.24517822265625, "learning_rate": 1.6845362108041811e-06, "loss": 0.2078, "reward": 0.125, "reward_std": 0.1369306445121765, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "step": 772 }, { "clip_ratio": 0.0, "completion_length": 621.7500152587891, "epoch": 0.5153333333333333, "grad_norm": 0.4192467927932739, "kl": 0.55023193359375, "learning_rate": 1.6810715719211231e-06, "loss": 0.0871, "reward": 0.3333333395421505, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.3333333395421505, "rewards/format_reward": 0.0, "step": 773 }, { "clip_ratio": 0.0, "completion_length": 399.6250114440918, "epoch": 0.516, "grad_norm": 0.287623792886734, "kl": 0.28887939453125, "learning_rate": 1.677605952459752e-06, "loss": 0.0136, "reward": 0.5833333395421505, "reward_std": 0.3680921792984009, "rewards/accuracy_reward": 0.5833333395421505, "rewards/format_reward": 0.0, "step": 774 }, { "clip_ratio": 0.0, "completion_length": 558.7916870117188, "epoch": 0.5166666666666667, "grad_norm": 1.4962282180786133, "kl": 1.2828369140625, "learning_rate": 1.6741393711878454e-06, "loss": 0.0612, "reward": 0.3333333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 775 }, { "clip_ratio": 0.0, "completion_length": 654.1250152587891, "epoch": 0.5173333333333333, "grad_norm": 0.4045711159706116, "kl": 0.8106689453125, "learning_rate": 1.6706718468783925e-06, "loss": 0.1417, "reward": 0.4166666716337204, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.4166666716337204, "rewards/format_reward": 0.0, "step": 776 }, { "clip_ratio": 0.0, "completion_length": 535.9166870117188, "epoch": 0.518, "grad_norm": 0.28047287464141846, "kl": 0.21966552734375, "learning_rate": 1.667203398309488e-06, "loss": 0.0267, "reward": 0.4166666865348816, "reward_std": 0.20412415266036987, "rewards/accuracy_reward": 0.4166666865348816, "rewards/format_reward": 0.0, "step": 777 }, { "clip_ratio": 0.0, "completion_length": 524.0000190734863, "epoch": 0.5186666666666667, "grad_norm": 1.3370088338851929, "kl": 1.8583984375, "learning_rate": 1.663734044264232e-06, "loss": 0.1616, "reward": 0.1250000037252903, "reward_std": 0.3061862140893936, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 778 }, { "clip_ratio": 0.0, "completion_length": 615.1250152587891, "epoch": 0.5193333333333333, "grad_norm": 0.7635619044303894, "kl": 1.67919921875, "learning_rate": 1.660263803530628e-06, "loss": 0.2029, "reward": 0.4166666753590107, "reward_std": 0.4971916303038597, "rewards/accuracy_reward": 0.4166666753590107, "rewards/format_reward": 0.0, "step": 779 }, { "clip_ratio": 0.0, "completion_length": 486.54168701171875, "epoch": 0.52, "grad_norm": 0.7878051996231079, "kl": 1.4178466796875, "learning_rate": 1.6567926949014804e-06, "loss": 0.0269, "reward": 0.3333333395421505, "reward_std": 0.3680921792984009, "rewards/accuracy_reward": 0.3333333395421505, "rewards/format_reward": 0.0, "step": 780 }, { "clip_ratio": 0.0, "completion_length": 495.9166793823242, "epoch": 0.5206666666666667, "grad_norm": 0.48201027512550354, "kl": 0.46429443359375, "learning_rate": 1.6533207371742955e-06, "loss": 0.1118, "reward": 0.3333333432674408, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 781 }, { "clip_ratio": 0.0, "completion_length": 523.3333435058594, "epoch": 0.5213333333333333, "grad_norm": 0.33032163977622986, "kl": 0.60198974609375, "learning_rate": 1.649847949151177e-06, "loss": 0.0152, "reward": 0.2500000074505806, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 782 }, { "clip_ratio": 0.0, "completion_length": 522.8333358764648, "epoch": 0.522, "grad_norm": 0.3140178620815277, "kl": 0.8599853515625, "learning_rate": 1.6463743496387243e-06, "loss": 0.1615, "reward": 0.5000000037252903, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.5000000037252903, "rewards/format_reward": 0.0, "step": 783 }, { "clip_ratio": 0.0, "completion_length": 600.5416946411133, "epoch": 0.5226666666666666, "grad_norm": 0.5788736343383789, "kl": 0.677001953125, "learning_rate": 1.6428999574479323e-06, "loss": 0.1443, "reward": 0.416666679084301, "reward_std": 0.44311685860157013, "rewards/accuracy_reward": 0.416666679084301, "rewards/format_reward": 0.0, "step": 784 }, { "clip_ratio": 0.0, "completion_length": 606.5833435058594, "epoch": 0.5233333333333333, "grad_norm": 0.44217702746391296, "kl": 1.24993896484375, "learning_rate": 1.6394247913940886e-06, "loss": 0.043, "reward": 0.291666679084301, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.291666679084301, "rewards/format_reward": 0.0, "step": 785 }, { "clip_ratio": 0.0, "completion_length": 594.0416870117188, "epoch": 0.524, "grad_norm": 0.43349215388298035, "kl": 0.5701904296875, "learning_rate": 1.6359488702966704e-06, "loss": 0.1332, "reward": 0.5000000111758709, "reward_std": 0.4779854267835617, "rewards/accuracy_reward": 0.5000000111758709, "rewards/format_reward": 0.0, "step": 786 }, { "clip_ratio": 0.0, "completion_length": 571.6250228881836, "epoch": 0.5246666666666666, "grad_norm": 0.41774389147758484, "kl": 0.318115234375, "learning_rate": 1.632472212979245e-06, "loss": -0.0327, "reward": 0.2083333395421505, "reward_std": 0.3602609932422638, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 787 }, { "clip_ratio": 0.0, "completion_length": 567.2916793823242, "epoch": 0.5253333333333333, "grad_norm": 0.9657918214797974, "kl": 1.092529296875, "learning_rate": 1.6289948382693663e-06, "loss": 0.0217, "reward": 0.2083333432674408, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.2083333432674408, "rewards/format_reward": 0.0, "step": 788 }, { "clip_ratio": 0.0, "completion_length": 561.6250076293945, "epoch": 0.526, "grad_norm": 0.38669073581695557, "kl": 1.4810791015625, "learning_rate": 1.6255167649984737e-06, "loss": 0.1108, "reward": 0.3750000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.3750000037252903, "rewards/format_reward": 0.0, "step": 789 }, { "clip_ratio": 0.0, "completion_length": 617.9583511352539, "epoch": 0.5266666666666666, "grad_norm": 3.5322535037994385, "kl": 1.047119140625, "learning_rate": 1.6220380120017874e-06, "loss": 0.052, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 790 }, { "clip_ratio": 0.0, "completion_length": 779.9583435058594, "epoch": 0.5273333333333333, "grad_norm": 0.7896983623504639, "kl": 1.80810546875, "learning_rate": 1.6185585981182121e-06, "loss": 0.0575, "reward": 0.1666666716337204, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 791 }, { "clip_ratio": 0.0, "completion_length": 525.9166793823242, "epoch": 0.528, "grad_norm": 0.2771357595920563, "kl": 0.73681640625, "learning_rate": 1.615078542190228e-06, "loss": 0.0305, "reward": 0.2500000074505806, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 792 }, { "clip_ratio": 0.0, "completion_length": 659.9166870117188, "epoch": 0.5286666666666666, "grad_norm": 1.4803900718688965, "kl": 1.548828125, "learning_rate": 1.6115978630637955e-06, "loss": 0.0907, "reward": 0.0833333358168602, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 793 }, { "clip_ratio": 0.0, "completion_length": 365.08334732055664, "epoch": 0.5293333333333333, "grad_norm": 0.5310041308403015, "kl": 0.1966552734375, "learning_rate": 1.6081165795882462e-06, "loss": 0.0628, "reward": 0.5, "reward_std": 0.273861289024353, "rewards/accuracy_reward": 0.5, "rewards/format_reward": 0.0, "step": 794 }, { "clip_ratio": 0.0, "completion_length": 716.7916717529297, "epoch": 0.53, "grad_norm": 0.5715692043304443, "kl": 0.191070556640625, "learning_rate": 1.6046347106161879e-06, "loss": 0.0287, "reward": 0.5416666753590107, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.5416666753590107, "rewards/format_reward": 0.0, "step": 795 }, { "clip_ratio": 0.0, "completion_length": 550.8750076293945, "epoch": 0.5306666666666666, "grad_norm": 0.624599814414978, "kl": 0.8251953125, "learning_rate": 1.6011522750033979e-06, "loss": 0.0396, "reward": 0.25, "reward_std": 0.0, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "step": 796 }, { "clip_ratio": 0.0, "completion_length": 650.0833511352539, "epoch": 0.5313333333333333, "grad_norm": 0.3773607313632965, "kl": 0.432373046875, "learning_rate": 1.5976692916087211e-06, "loss": 0.0636, "reward": 0.2083333358168602, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.2083333358168602, "rewards/format_reward": 0.0, "step": 797 }, { "clip_ratio": 0.0, "completion_length": 555.2500076293945, "epoch": 0.532, "grad_norm": 0.3984975814819336, "kl": 0.49267578125, "learning_rate": 1.5941857792939703e-06, "loss": 0.0853, "reward": 0.2500000074505806, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 798 }, { "clip_ratio": 0.0, "completion_length": 657.2500152587891, "epoch": 0.5326666666666666, "grad_norm": 0.6008354425430298, "kl": 0.417236328125, "learning_rate": 1.5907017569238214e-06, "loss": 0.0926, "reward": 0.1666666716337204, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 799 }, { "clip_ratio": 0.0, "completion_length": 746.0000305175781, "epoch": 0.5333333333333333, "grad_norm": 0.5225312113761902, "kl": 0.77099609375, "learning_rate": 1.5872172433657137e-06, "loss": 0.035, "reward": 0.0833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 800 }, { "clip_ratio": 0.0, "completion_length": 696.6666870117188, "epoch": 0.534, "grad_norm": 0.3115634322166443, "kl": 0.9697265625, "learning_rate": 1.5837322574897458e-06, "loss": 0.0703, "reward": 0.0833333358168602, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 801 }, { "clip_ratio": 0.0, "completion_length": 518.4166793823242, "epoch": 0.5346666666666666, "grad_norm": 2.10563325881958, "kl": 0.5390625, "learning_rate": 1.5802468181685739e-06, "loss": 0.2009, "reward": 0.4583333544433117, "reward_std": 0.306186206638813, "rewards/accuracy_reward": 0.4583333544433117, "rewards/format_reward": 0.0, "step": 802 }, { "clip_ratio": 0.0, "completion_length": 384.0833435058594, "epoch": 0.5353333333333333, "grad_norm": 1.470822811126709, "kl": 0.29620361328125, "learning_rate": 1.5767609442773104e-06, "loss": 0.2372, "reward": 0.6250000223517418, "reward_std": 0.4701542407274246, "rewards/accuracy_reward": 0.6250000223517418, "rewards/format_reward": 0.0, "step": 803 }, { "clip_ratio": 0.0, "completion_length": 671.9166793823242, "epoch": 0.536, "grad_norm": 0.39356693625450134, "kl": 0.90771484375, "learning_rate": 1.5732746546934201e-06, "loss": 0.1309, "reward": 0.2083333358168602, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.2083333358168602, "rewards/format_reward": 0.0, "step": 804 }, { "clip_ratio": 0.0, "completion_length": 582.7500305175781, "epoch": 0.5366666666666666, "grad_norm": 1.0127586126327515, "kl": 1.1044921875, "learning_rate": 1.5697879682966203e-06, "loss": 0.1284, "reward": 0.25, "reward_std": 0.273861289024353, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "step": 805 }, { "clip_ratio": 0.0, "completion_length": 616.3333435058594, "epoch": 0.5373333333333333, "grad_norm": 0.4048437178134918, "kl": 0.48370361328125, "learning_rate": 1.5663009039687759e-06, "loss": 0.0218, "reward": 0.5833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.5833333358168602, "rewards/format_reward": 0.0, "step": 806 }, { "clip_ratio": 0.0, "completion_length": 555.2083587646484, "epoch": 0.538, "grad_norm": 0.547532856464386, "kl": 0.6103515625, "learning_rate": 1.5628134805937994e-06, "loss": 0.0938, "reward": 0.2916666716337204, "reward_std": 0.39512956142425537, "rewards/accuracy_reward": 0.2916666716337204, "rewards/format_reward": 0.0, "step": 807 }, { "clip_ratio": 0.0, "completion_length": 508.291690826416, "epoch": 0.5386666666666666, "grad_norm": 0.325888067483902, "kl": 0.71826171875, "learning_rate": 1.5593257170575481e-06, "loss": 0.0408, "reward": 0.2916666716337204, "reward_std": 0.39512956142425537, "rewards/accuracy_reward": 0.2916666716337204, "rewards/format_reward": 0.0, "step": 808 }, { "clip_ratio": 0.0, "completion_length": 678.4583587646484, "epoch": 0.5393333333333333, "grad_norm": 0.24012231826782227, "kl": 0.591796875, "learning_rate": 1.555837632247719e-06, "loss": 0.0251, "reward": 0.0833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 809 }, { "clip_ratio": 0.0, "completion_length": 539.3750152587891, "epoch": 0.54, "grad_norm": 1.0212724208831787, "kl": 0.61456298828125, "learning_rate": 1.5523492450537518e-06, "loss": 0.3456, "reward": 0.6666666716337204, "reward_std": 0.40296074748039246, "rewards/accuracy_reward": 0.6666666716337204, "rewards/format_reward": 0.0, "step": 810 }, { "clip_ratio": 0.0, "completion_length": 596.2916717529297, "epoch": 0.5406666666666666, "grad_norm": 0.6341525316238403, "kl": 0.44091796875, "learning_rate": 1.5488605743667219e-06, "loss": 0.0102, "reward": 0.5000000111758709, "reward_std": 0.4779854342341423, "rewards/accuracy_reward": 0.5000000111758709, "rewards/format_reward": 0.0, "step": 811 }, { "clip_ratio": 0.0, "completion_length": 720.0000152587891, "epoch": 0.5413333333333333, "grad_norm": 0.4842911660671234, "kl": 1.4208984375, "learning_rate": 1.5453716390792412e-06, "loss": 0.2034, "reward": 0.1250000037252903, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 812 }, { "clip_ratio": 0.0, "completion_length": 736.5833587646484, "epoch": 0.542, "grad_norm": 0.37966111302375793, "kl": 1.25732421875, "learning_rate": 1.5418824580853535e-06, "loss": 0.1483, "reward": 0.1666666716337204, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 813 }, { "clip_ratio": 0.0, "completion_length": 535.1666870117188, "epoch": 0.5426666666666666, "grad_norm": 0.359665185213089, "kl": 0.586669921875, "learning_rate": 1.5383930502804333e-06, "loss": 0.1708, "reward": 0.5833333395421505, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.5833333395421505, "rewards/format_reward": 0.0, "step": 814 }, { "clip_ratio": 0.0, "completion_length": 720.2083435058594, "epoch": 0.5433333333333333, "grad_norm": 0.7152715921401978, "kl": 0.892578125, "learning_rate": 1.5349034345610844e-06, "loss": 0.1017, "reward": 0.0416666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 815 }, { "clip_ratio": 0.0, "completion_length": 613.7916793823242, "epoch": 0.544, "grad_norm": 0.337433397769928, "kl": 0.62762451171875, "learning_rate": 1.5314136298250356e-06, "loss": 0.0823, "reward": 0.3333333432674408, "reward_std": 0.46232305467128754, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 816 }, { "clip_ratio": 0.0, "completion_length": 600.4166717529297, "epoch": 0.5446666666666666, "grad_norm": 0.5324622988700867, "kl": 1.560546875, "learning_rate": 1.5279236549710393e-06, "loss": 0.1885, "reward": 0.2500000037252903, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.0, "step": 817 }, { "clip_ratio": 0.0, "completion_length": 503.4166793823242, "epoch": 0.5453333333333333, "grad_norm": 1.0061956644058228, "kl": 1.4410400390625, "learning_rate": 1.5244335288987687e-06, "loss": 0.1469, "reward": 0.5416666828095913, "reward_std": 0.4701542481780052, "rewards/accuracy_reward": 0.5416666828095913, "rewards/format_reward": 0.0, "step": 818 }, { "clip_ratio": 0.0, "completion_length": 567.3750114440918, "epoch": 0.546, "grad_norm": 1.1555522680282593, "kl": 1.2716064453125, "learning_rate": 1.5209432705087182e-06, "loss": 0.0551, "reward": 0.2083333432674408, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.2083333432674408, "rewards/format_reward": 0.0, "step": 819 }, { "clip_ratio": 0.0, "completion_length": 470.91668701171875, "epoch": 0.5466666666666666, "grad_norm": 0.1778678297996521, "kl": 0.201904296875, "learning_rate": 1.5174528987020958e-06, "loss": 0.0088, "reward": 0.25, "reward_std": 0.0, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "step": 820 }, { "clip_ratio": 0.0, "completion_length": 550.1666793823242, "epoch": 0.5473333333333333, "grad_norm": 0.42744341492652893, "kl": 0.566650390625, "learning_rate": 1.5139624323807253e-06, "loss": 0.11, "reward": 0.2083333395421505, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 821 }, { "clip_ratio": 0.0, "completion_length": 547.3750076293945, "epoch": 0.548, "grad_norm": 0.35106754302978516, "kl": 0.712158203125, "learning_rate": 1.5104718904469423e-06, "loss": 0.0373, "reward": 0.1250000037252903, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 822 }, { "clip_ratio": 0.0, "completion_length": 567.9166870117188, "epoch": 0.5486666666666666, "grad_norm": 0.6810794472694397, "kl": 1.19921875, "learning_rate": 1.506981291803492e-06, "loss": 0.2189, "reward": 0.3750000111758709, "reward_std": 0.3410547822713852, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 823 }, { "clip_ratio": 0.0, "completion_length": 570.8750228881836, "epoch": 0.5493333333333333, "grad_norm": 0.32110267877578735, "kl": 0.489990234375, "learning_rate": 1.5034906553534257e-06, "loss": 0.0421, "reward": 0.5416666679084301, "reward_std": 0.37592336535453796, "rewards/accuracy_reward": 0.5416666679084301, "rewards/format_reward": 0.0, "step": 824 }, { "clip_ratio": 0.0, "completion_length": 769.5833435058594, "epoch": 0.55, "grad_norm": 0.5387272834777832, "kl": 0.977783203125, "learning_rate": 1.5e-06, "loss": 0.0663, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 825 }, { "clip_ratio": 0.0, "completion_length": 407.2500114440918, "epoch": 0.5506666666666666, "grad_norm": 0.657080888748169, "kl": 0.6953125, "learning_rate": 1.4965093446465748e-06, "loss": -0.018, "reward": 0.3333333469927311, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.3333333469927311, "rewards/format_reward": 0.0, "step": 826 }, { "clip_ratio": 0.0, "completion_length": 514.1666793823242, "epoch": 0.5513333333333333, "grad_norm": 0.5971965789794922, "kl": 0.3289794921875, "learning_rate": 1.4930187081965083e-06, "loss": 0.1074, "reward": 0.3333333432674408, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 827 }, { "clip_ratio": 0.0, "completion_length": 576.5000152587891, "epoch": 0.552, "grad_norm": 1.2446520328521729, "kl": 0.75927734375, "learning_rate": 1.4895281095530578e-06, "loss": 0.2501, "reward": 0.5416666865348816, "reward_std": 0.3410547897219658, "rewards/accuracy_reward": 0.5416666865348816, "rewards/format_reward": 0.0, "step": 828 }, { "clip_ratio": 0.0, "completion_length": 663.0416679382324, "epoch": 0.5526666666666666, "grad_norm": 0.5984786748886108, "kl": 0.92510986328125, "learning_rate": 1.486037567619275e-06, "loss": 0.037, "reward": 0.5, "reward_std": 0.0, "rewards/accuracy_reward": 0.5, "rewards/format_reward": 0.0, "step": 829 }, { "clip_ratio": 0.0, "completion_length": 449.29168701171875, "epoch": 0.5533333333333333, "grad_norm": 0.1590639054775238, "kl": 0.214752197265625, "learning_rate": 1.4825471012979047e-06, "loss": 0.0116, "reward": 0.25, "reward_std": 0.0, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "step": 830 }, { "clip_ratio": 0.0, "completion_length": 659.0416793823242, "epoch": 0.554, "grad_norm": 0.3568677008152008, "kl": 0.828521728515625, "learning_rate": 1.4790567294912823e-06, "loss": 0.0994, "reward": 0.2916666716337204, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.2916666716337204, "rewards/format_reward": 0.0, "step": 831 }, { "clip_ratio": 0.0, "completion_length": 563.5833435058594, "epoch": 0.5546666666666666, "grad_norm": 0.8327547311782837, "kl": 0.65203857421875, "learning_rate": 1.4755664711012314e-06, "loss": 0.0764, "reward": 0.3333333469927311, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.3333333469927311, "rewards/format_reward": 0.0, "step": 832 }, { "clip_ratio": 0.0, "completion_length": 660.3750152587891, "epoch": 0.5553333333333333, "grad_norm": 0.6476672291755676, "kl": 1.0625, "learning_rate": 1.4720763450289612e-06, "loss": 0.2307, "reward": 0.1666666679084301, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 833 }, { "clip_ratio": 0.0, "completion_length": 734.2500305175781, "epoch": 0.556, "grad_norm": 0.6333141922950745, "kl": 1.0341796875, "learning_rate": 1.4685863701749647e-06, "loss": 0.0364, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 834 }, { "clip_ratio": 0.0, "completion_length": 434.04167556762695, "epoch": 0.5566666666666666, "grad_norm": 0.4225279688835144, "kl": 0.71502685546875, "learning_rate": 1.4650965654389156e-06, "loss": 0.1293, "reward": 0.5416666716337204, "reward_std": 0.39512956142425537, "rewards/accuracy_reward": 0.5416666716337204, "rewards/format_reward": 0.0, "step": 835 }, { "clip_ratio": 0.0, "completion_length": 677.1250152587891, "epoch": 0.5573333333333333, "grad_norm": 0.9128155708312988, "kl": 1.83203125, "learning_rate": 1.461606949719567e-06, "loss": 0.1766, "reward": 0.2083333395421505, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 836 }, { "clip_ratio": 0.0, "completion_length": 725.1666870117188, "epoch": 0.558, "grad_norm": 0.48559901118278503, "kl": 1.3984375, "learning_rate": 1.458117541914647e-06, "loss": 0.2842, "reward": 0.1666666716337204, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 837 }, { "clip_ratio": 0.0, "completion_length": 386.62500762939453, "epoch": 0.5586666666666666, "grad_norm": 0.8375260233879089, "kl": 0.75103759765625, "learning_rate": 1.454628360920759e-06, "loss": 0.1815, "reward": 0.6666666828095913, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.6666666828095913, "rewards/format_reward": 0.0, "step": 838 }, { "clip_ratio": 0.0, "completion_length": 776.5416793823242, "epoch": 0.5593333333333333, "grad_norm": 0.4752759337425232, "kl": 1.6298828125, "learning_rate": 1.4511394256332784e-06, "loss": 0.1832, "reward": 0.2500000111758709, "reward_std": 0.20412415266036987, "rewards/accuracy_reward": 0.2500000111758709, "rewards/format_reward": 0.0, "step": 839 }, { "clip_ratio": 0.0, "completion_length": 657.6250152587891, "epoch": 0.56, "grad_norm": 0.8674920797348022, "kl": 0.81787109375, "learning_rate": 1.4476507549462489e-06, "loss": 0.1834, "reward": 0.375, "reward_std": 0.41079193353652954, "rewards/accuracy_reward": 0.375, "rewards/format_reward": 0.0, "step": 840 }, { "clip_ratio": 0.0, "completion_length": 659.9583587646484, "epoch": 0.5606666666666666, "grad_norm": 0.46731051802635193, "kl": 1.1395263671875, "learning_rate": 1.4441623677522811e-06, "loss": 0.0839, "reward": 0.2083333432674408, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.2083333432674408, "rewards/format_reward": 0.0, "step": 841 }, { "clip_ratio": 0.0, "completion_length": 665.5417022705078, "epoch": 0.5613333333333334, "grad_norm": 0.3469387888908386, "kl": 1.36328125, "learning_rate": 1.4406742829424524e-06, "loss": 0.1673, "reward": 0.2916666679084301, "reward_std": 0.37592335790395737, "rewards/accuracy_reward": 0.2916666679084301, "rewards/format_reward": 0.0, "step": 842 }, { "clip_ratio": 0.0, "completion_length": 538.9166717529297, "epoch": 0.562, "grad_norm": 0.6619201898574829, "kl": 0.3614501953125, "learning_rate": 1.4371865194062009e-06, "loss": 0.0551, "reward": 0.2500000111758709, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.2500000111758709, "rewards/format_reward": 0.0, "step": 843 }, { "clip_ratio": 0.0, "completion_length": 632.1250228881836, "epoch": 0.5626666666666666, "grad_norm": 0.4512653648853302, "kl": 0.74560546875, "learning_rate": 1.433699096031224e-06, "loss": 0.0859, "reward": 0.4583333507180214, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.4583333507180214, "rewards/format_reward": 0.0, "step": 844 }, { "clip_ratio": 0.0, "completion_length": 467.50001525878906, "epoch": 0.5633333333333334, "grad_norm": 0.7436599731445312, "kl": 0.73291015625, "learning_rate": 1.43021203170338e-06, "loss": 0.0251, "reward": 0.3750000111758709, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 845 }, { "clip_ratio": 0.0, "completion_length": 350.6666793823242, "epoch": 0.564, "grad_norm": 0.7599238157272339, "kl": 0.53662109375, "learning_rate": 1.4267253453065802e-06, "loss": 0.0909, "reward": 0.4166666865348816, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.4166666865348816, "rewards/format_reward": 0.0, "step": 846 }, { "clip_ratio": 0.0, "completion_length": 733.9166870117188, "epoch": 0.5646666666666667, "grad_norm": 0.9028979539871216, "kl": 1.353515625, "learning_rate": 1.4232390557226905e-06, "loss": 0.0406, "reward": 0.1666666679084301, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 847 }, { "clip_ratio": 0.0, "completion_length": 312.8750114440918, "epoch": 0.5653333333333334, "grad_norm": 0.5691962838172913, "kl": 0.30841064453125, "learning_rate": 1.4197531818314262e-06, "loss": 0.1201, "reward": 0.8750000149011612, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.8750000149011612, "rewards/format_reward": 0.0, "step": 848 }, { "clip_ratio": 0.0, "completion_length": 663.6250152587891, "epoch": 0.566, "grad_norm": 0.4753604829311371, "kl": 1.068359375, "learning_rate": 1.4162677425102543e-06, "loss": 0.2152, "reward": 0.3333333469927311, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.3333333469927311, "rewards/format_reward": 0.0, "step": 849 }, { "clip_ratio": 0.0, "completion_length": 598.2083587646484, "epoch": 0.5666666666666667, "grad_norm": 0.4323820173740387, "kl": 0.75927734375, "learning_rate": 1.4127827566342864e-06, "loss": 0.0301, "reward": 0.375, "reward_std": 0.1369306445121765, "rewards/accuracy_reward": 0.375, "rewards/format_reward": 0.0, "step": 850 }, { "clip_ratio": 0.0, "completion_length": 632.5833435058594, "epoch": 0.5673333333333334, "grad_norm": 0.6575254797935486, "kl": 0.8046875, "learning_rate": 1.4092982430761784e-06, "loss": 0.0333, "reward": 0.25, "reward_std": 0.0, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "step": 851 }, { "clip_ratio": 0.0, "completion_length": 579.4166793823242, "epoch": 0.568, "grad_norm": 0.4137823283672333, "kl": 0.68701171875, "learning_rate": 1.40581422070603e-06, "loss": 0.1571, "reward": 0.3750000149011612, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.3750000149011612, "rewards/format_reward": 0.0, "step": 852 }, { "clip_ratio": 0.0, "completion_length": 488.4166793823242, "epoch": 0.5686666666666667, "grad_norm": 0.3689560890197754, "kl": 0.4228515625, "learning_rate": 1.4023307083912792e-06, "loss": 0.0345, "reward": 0.3333333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 853 }, { "clip_ratio": 0.0, "completion_length": 570.3750152587891, "epoch": 0.5693333333333334, "grad_norm": 0.39705029129981995, "kl": 0.5338134765625, "learning_rate": 1.3988477249966026e-06, "loss": 0.0734, "reward": 0.4583333544433117, "reward_std": 0.3061862215399742, "rewards/accuracy_reward": 0.4583333544433117, "rewards/format_reward": 0.0, "step": 854 }, { "clip_ratio": 0.0, "completion_length": 614.2083511352539, "epoch": 0.57, "grad_norm": 0.4996829032897949, "kl": 0.602996826171875, "learning_rate": 1.395365289383812e-06, "loss": 0.0673, "reward": 0.4166666716337204, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.4166666716337204, "rewards/format_reward": 0.0, "step": 855 }, { "clip_ratio": 0.0, "completion_length": 702.6250076293945, "epoch": 0.5706666666666667, "grad_norm": 0.4088698923587799, "kl": 0.405517578125, "learning_rate": 1.391883420411754e-06, "loss": 0.079, "reward": 0.2083333432674408, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.2083333432674408, "rewards/format_reward": 0.0, "step": 856 }, { "clip_ratio": 0.0, "completion_length": 688.8750152587891, "epoch": 0.5713333333333334, "grad_norm": 0.6513028740882874, "kl": 0.9072265625, "learning_rate": 1.3884021369362052e-06, "loss": 0.0702, "reward": 0.4166666753590107, "reward_std": 0.4971916303038597, "rewards/accuracy_reward": 0.4166666753590107, "rewards/format_reward": 0.0, "step": 857 }, { "clip_ratio": 0.0, "completion_length": 609.5833587646484, "epoch": 0.572, "grad_norm": 0.5809162855148315, "kl": 0.4189453125, "learning_rate": 1.384921457809772e-06, "loss": 0.1581, "reward": 0.3750000111758709, "reward_std": 0.3602609932422638, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 858 }, { "clip_ratio": 0.0, "completion_length": 305.29167556762695, "epoch": 0.5726666666666667, "grad_norm": 0.339732825756073, "kl": 0.27081298828125, "learning_rate": 1.3814414018817884e-06, "loss": 0.02, "reward": 0.7083333432674408, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.7083333432674408, "rewards/format_reward": 0.0, "step": 859 }, { "clip_ratio": 0.0, "completion_length": 563.7083511352539, "epoch": 0.5733333333333334, "grad_norm": 0.4411391019821167, "kl": 0.73431396484375, "learning_rate": 1.3779619879982127e-06, "loss": -0.0126, "reward": 0.2500000074505806, "reward_std": 0.3332236111164093, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 860 }, { "clip_ratio": 0.0, "completion_length": 502.75000381469727, "epoch": 0.574, "grad_norm": 0.5527832508087158, "kl": 0.67694091796875, "learning_rate": 1.374483235001527e-06, "loss": 0.0103, "reward": 0.2500000111758709, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.2500000111758709, "rewards/format_reward": 0.0, "step": 861 }, { "clip_ratio": 0.0, "completion_length": 713.7083511352539, "epoch": 0.5746666666666667, "grad_norm": 3.6889679431915283, "kl": 2.05029296875, "learning_rate": 1.3710051617306337e-06, "loss": 0.0985, "reward": 0.0833333358168602, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 862 }, { "clip_ratio": 0.0, "completion_length": 609.0416870117188, "epoch": 0.5753333333333334, "grad_norm": 0.8651523590087891, "kl": 0.41748046875, "learning_rate": 1.3675277870207552e-06, "loss": 0.1613, "reward": 0.5000000223517418, "reward_std": 0.40824829041957855, "rewards/accuracy_reward": 0.5000000223517418, "rewards/format_reward": 0.0, "step": 863 }, { "clip_ratio": 0.0, "completion_length": 683.2083511352539, "epoch": 0.576, "grad_norm": 0.38036367297172546, "kl": 0.221923828125, "learning_rate": 1.36405112970333e-06, "loss": -0.0251, "reward": 0.1666666716337204, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 864 }, { "clip_ratio": 0.0, "completion_length": 660.6666946411133, "epoch": 0.5766666666666667, "grad_norm": 0.29805871844291687, "kl": 0.48663330078125, "learning_rate": 1.3605752086059117e-06, "loss": 0.0738, "reward": 0.1666666679084301, "reward_std": 0.23899272084236145, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 865 }, { "clip_ratio": 0.0, "completion_length": 523.4583435058594, "epoch": 0.5773333333333334, "grad_norm": 0.4100269675254822, "kl": 0.3778076171875, "learning_rate": 1.3571000425520677e-06, "loss": 0.0541, "reward": 0.2083333432674408, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.2083333432674408, "rewards/format_reward": 0.0, "step": 866 }, { "clip_ratio": 0.0, "completion_length": 531.3333358764648, "epoch": 0.578, "grad_norm": 0.5020498037338257, "kl": 0.58648681640625, "learning_rate": 1.353625650361276e-06, "loss": 0.0406, "reward": 0.2083333395421505, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 867 }, { "clip_ratio": 0.0, "completion_length": 638.9166870117188, "epoch": 0.5786666666666667, "grad_norm": 0.8084805607795715, "kl": 0.8720703125, "learning_rate": 1.3501520508488237e-06, "loss": 0.0328, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 868 }, { "clip_ratio": 0.0, "completion_length": 539.9166793823242, "epoch": 0.5793333333333334, "grad_norm": 0.7555376291275024, "kl": 0.681640625, "learning_rate": 1.3466792628257048e-06, "loss": 0.119, "reward": 0.2916666679084301, "reward_std": 0.37592335790395737, "rewards/accuracy_reward": 0.2916666679084301, "rewards/format_reward": 0.0, "step": 869 }, { "clip_ratio": 0.0, "completion_length": 454.9166831970215, "epoch": 0.58, "grad_norm": 0.31955111026763916, "kl": 0.271240234375, "learning_rate": 1.3432073050985201e-06, "loss": -0.033, "reward": 0.4583333395421505, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.4583333395421505, "rewards/format_reward": 0.0, "step": 870 }, { "clip_ratio": 0.0, "completion_length": 552.7083435058594, "epoch": 0.5806666666666667, "grad_norm": 0.5216869115829468, "kl": 0.41754150390625, "learning_rate": 1.3397361964693728e-06, "loss": 0.066, "reward": 0.3750000074505806, "reward_std": 0.4701542481780052, "rewards/accuracy_reward": 0.3750000074505806, "rewards/format_reward": 0.0, "step": 871 }, { "clip_ratio": 0.0, "completion_length": 445.1666793823242, "epoch": 0.5813333333333334, "grad_norm": 0.27342209219932556, "kl": 0.3349609375, "learning_rate": 1.3362659557357678e-06, "loss": 0.0276, "reward": 0.2500000074505806, "reward_std": 0.3872983753681183, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 872 }, { "clip_ratio": 0.0, "completion_length": 590.6666946411133, "epoch": 0.582, "grad_norm": 3.7706732749938965, "kl": 0.570068359375, "learning_rate": 1.332796601690512e-06, "loss": 0.0695, "reward": 0.3333333432674408, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 873 }, { "clip_ratio": 0.0, "completion_length": 719.1666717529297, "epoch": 0.5826666666666667, "grad_norm": 0.675914466381073, "kl": 1.1005859375, "learning_rate": 1.3293281531216078e-06, "loss": 0.0321, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 874 }, { "clip_ratio": 0.0, "completion_length": 474.75001525878906, "epoch": 0.5833333333333334, "grad_norm": 0.47885704040527344, "kl": 0.5927734375, "learning_rate": 1.3258606288121545e-06, "loss": 0.1356, "reward": 0.416666679084301, "reward_std": 0.3872983753681183, "rewards/accuracy_reward": 0.416666679084301, "rewards/format_reward": 0.0, "step": 875 }, { "clip_ratio": 0.0, "completion_length": 488.62500762939453, "epoch": 0.584, "grad_norm": 0.28345444798469543, "kl": 0.3612060546875, "learning_rate": 1.3223940475402486e-06, "loss": 0.0588, "reward": 0.541666679084301, "reward_std": 0.3061862140893936, "rewards/accuracy_reward": 0.541666679084301, "rewards/format_reward": 0.0, "step": 876 }, { "clip_ratio": 0.0, "completion_length": 418.50000762939453, "epoch": 0.5846666666666667, "grad_norm": 0.5224109888076782, "kl": 0.720458984375, "learning_rate": 1.3189284280788772e-06, "loss": 0.1572, "reward": 0.4583333395421505, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.4583333395421505, "rewards/format_reward": 0.0, "step": 877 }, { "clip_ratio": 0.0, "completion_length": 548.4583435058594, "epoch": 0.5853333333333334, "grad_norm": 0.5054531097412109, "kl": 0.58984375, "learning_rate": 1.3154637891958194e-06, "loss": 0.0938, "reward": 0.4166666753590107, "reward_std": 0.4971916303038597, "rewards/accuracy_reward": 0.4166666753590107, "rewards/format_reward": 0.0, "step": 878 }, { "clip_ratio": 0.0, "completion_length": 319.4583435058594, "epoch": 0.586, "grad_norm": 0.5215534567832947, "kl": 0.349853515625, "learning_rate": 1.3120001496535434e-06, "loss": 0.0959, "reward": 0.7083333544433117, "reward_std": 0.3061862140893936, "rewards/accuracy_reward": 0.7083333544433117, "rewards/format_reward": 0.0, "step": 879 }, { "clip_ratio": 0.0, "completion_length": 451.1250190734863, "epoch": 0.5866666666666667, "grad_norm": 12.836323738098145, "kl": 0.32769775390625, "learning_rate": 1.308537528209108e-06, "loss": -0.0062, "reward": 0.2083333395421505, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 880 }, { "clip_ratio": 0.0, "completion_length": 596.5833511352539, "epoch": 0.5873333333333334, "grad_norm": 0.44508472084999084, "kl": 0.73486328125, "learning_rate": 1.3050759436140566e-06, "loss": 0.0815, "reward": 0.2083333395421505, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 881 }, { "clip_ratio": 0.0, "completion_length": 570.0416870117188, "epoch": 0.588, "grad_norm": 0.4147035479545593, "kl": 0.484130859375, "learning_rate": 1.3016154146143161e-06, "loss": -0.0167, "reward": 0.1250000037252903, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 882 }, { "clip_ratio": 0.0, "completion_length": 560.8750152587891, "epoch": 0.5886666666666667, "grad_norm": 0.43442025780677795, "kl": 0.956390380859375, "learning_rate": 1.2981559599501011e-06, "loss": 0.0389, "reward": 0.3333333432674408, "reward_std": 0.3872983753681183, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 883 }, { "clip_ratio": 0.0, "completion_length": 475.4166793823242, "epoch": 0.5893333333333334, "grad_norm": 0.31292930245399475, "kl": 0.8455810546875, "learning_rate": 1.294697598355805e-06, "loss": 0.1166, "reward": 0.4166666679084301, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.4166666679084301, "rewards/format_reward": 0.0, "step": 884 }, { "clip_ratio": 0.0, "completion_length": 519.9166793823242, "epoch": 0.59, "grad_norm": 0.48729896545410156, "kl": 1.0340576171875, "learning_rate": 1.2912403485599022e-06, "loss": 0.0302, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 885 }, { "clip_ratio": 0.0, "completion_length": 620.8333587646484, "epoch": 0.5906666666666667, "grad_norm": 0.3456827700138092, "kl": 0.64739990234375, "learning_rate": 1.2877842292848466e-06, "loss": 0.1431, "reward": 0.2916666716337204, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.2916666716337204, "rewards/format_reward": 0.0, "step": 886 }, { "clip_ratio": 0.0, "completion_length": 747.5416870117188, "epoch": 0.5913333333333334, "grad_norm": 0.6261999011039734, "kl": 0.5654296875, "learning_rate": 1.284329259246971e-06, "loss": 0.0322, "reward": 0.1666666679084301, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 887 }, { "clip_ratio": 0.0, "completion_length": 474.3333435058594, "epoch": 0.592, "grad_norm": 0.2500125765800476, "kl": 0.1912841796875, "learning_rate": 1.2808754571563827e-06, "loss": 0.0141, "reward": 0.2083333358168602, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.2083333358168602, "rewards/format_reward": 0.0, "step": 888 }, { "clip_ratio": 0.0, "completion_length": 425.62501525878906, "epoch": 0.5926666666666667, "grad_norm": 0.722004234790802, "kl": 0.576171875, "learning_rate": 1.277422841716865e-06, "loss": 0.1287, "reward": 0.4166666716337204, "reward_std": 0.40296074748039246, "rewards/accuracy_reward": 0.4166666716337204, "rewards/format_reward": 0.0, "step": 889 }, { "clip_ratio": 0.0, "completion_length": 426.70834732055664, "epoch": 0.5933333333333334, "grad_norm": 0.5124059915542603, "kl": 0.7464599609375, "learning_rate": 1.2739714316257753e-06, "loss": 0.0287, "reward": 0.3750000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.3750000037252903, "rewards/format_reward": 0.0, "step": 890 }, { "clip_ratio": 0.0, "completion_length": 577.3333587646484, "epoch": 0.594, "grad_norm": 0.4983215928077698, "kl": 0.729248046875, "learning_rate": 1.2705212455739432e-06, "loss": 0.1358, "reward": 0.2500000037252903, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.0, "step": 891 }, { "clip_ratio": 0.0, "completion_length": 618.7500305175781, "epoch": 0.5946666666666667, "grad_norm": 0.3501860201358795, "kl": 0.501953125, "learning_rate": 1.2670723022455688e-06, "loss": 0.0567, "reward": 0.2916666753590107, "reward_std": 0.3602609932422638, "rewards/accuracy_reward": 0.2916666753590107, "rewards/format_reward": 0.0, "step": 892 }, { "clip_ratio": 0.0, "completion_length": 416.7916793823242, "epoch": 0.5953333333333334, "grad_norm": 0.2748178243637085, "kl": 0.1851806640625, "learning_rate": 1.2636246203181236e-06, "loss": -0.0036, "reward": 0.541666679084301, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.541666679084301, "rewards/format_reward": 0.0, "step": 893 }, { "clip_ratio": 0.0, "completion_length": 589.2500228881836, "epoch": 0.596, "grad_norm": 0.6685190200805664, "kl": 0.673583984375, "learning_rate": 1.260178218462248e-06, "loss": 0.1216, "reward": 0.1666666679084301, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 894 }, { "clip_ratio": 0.0, "completion_length": 591.5833587646484, "epoch": 0.5966666666666667, "grad_norm": 0.4964500665664673, "kl": 0.5322265625, "learning_rate": 1.256733115341649e-06, "loss": 0.0598, "reward": 0.125, "reward_std": 0.1369306445121765, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "step": 895 }, { "clip_ratio": 0.0, "completion_length": 616.7500152587891, "epoch": 0.5973333333333334, "grad_norm": 0.34750646352767944, "kl": 0.3968505859375, "learning_rate": 1.2532893296130016e-06, "loss": 0.0177, "reward": 0.25, "reward_std": 0.0, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "step": 896 }, { "clip_ratio": 0.0, "completion_length": 451.7500114440918, "epoch": 0.598, "grad_norm": 0.43368905782699585, "kl": 0.6234130859375, "learning_rate": 1.2498468799258468e-06, "loss": 0.0277, "reward": 0.2083333395421505, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 897 }, { "clip_ratio": 0.0, "completion_length": 579.7500305175781, "epoch": 0.5986666666666667, "grad_norm": 0.37874141335487366, "kl": 0.83245849609375, "learning_rate": 1.246405784922489e-06, "loss": 0.0487, "reward": 0.1250000037252903, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 898 }, { "clip_ratio": 0.0, "completion_length": 391.50000762939453, "epoch": 0.5993333333333334, "grad_norm": 1.198349952697754, "kl": 0.47198486328125, "learning_rate": 1.242966063237898e-06, "loss": 0.1031, "reward": 0.7083333544433117, "reward_std": 0.306186206638813, "rewards/accuracy_reward": 0.7083333544433117, "rewards/format_reward": 0.0, "step": 899 }, { "clip_ratio": 0.0, "completion_length": 624.1250228881836, "epoch": 0.6, "grad_norm": 0.2713553309440613, "kl": 0.240478515625, "learning_rate": 1.2395277334996047e-06, "loss": 0.047, "reward": 0.2916666753590107, "reward_std": 0.3602609932422638, "rewards/accuracy_reward": 0.2916666753590107, "rewards/format_reward": 0.0, "step": 900 }, { "clip_ratio": 0.0, "completion_length": 540.6666870117188, "epoch": 0.6006666666666667, "grad_norm": 0.5263336300849915, "kl": 0.8785400390625, "learning_rate": 1.2360908143276038e-06, "loss": 0.0632, "reward": 0.5416666679084301, "reward_std": 0.37592335790395737, "rewards/accuracy_reward": 0.5416666679084301, "rewards/format_reward": 0.0, "step": 901 }, { "clip_ratio": 0.0, "completion_length": 575.7083587646484, "epoch": 0.6013333333333334, "grad_norm": 0.35222455859184265, "kl": 0.41973876953125, "learning_rate": 1.2326553243342506e-06, "loss": 0.0574, "reward": 0.5000000037252903, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.5000000037252903, "rewards/format_reward": 0.0, "step": 902 }, { "clip_ratio": 0.0, "completion_length": 663.4583435058594, "epoch": 0.602, "grad_norm": 0.5595137476921082, "kl": 1.1689453125, "learning_rate": 1.2292212821241601e-06, "loss": 0.1026, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 903 }, { "clip_ratio": 0.0, "completion_length": 617.0416870117188, "epoch": 0.6026666666666667, "grad_norm": 0.28891706466674805, "kl": 0.73583984375, "learning_rate": 1.2257887062941081e-06, "loss": 0.038, "reward": 0.2500000074505806, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 904 }, { "clip_ratio": 0.0, "completion_length": 631.8333435058594, "epoch": 0.6033333333333334, "grad_norm": 0.3416825234889984, "kl": 0.48974609375, "learning_rate": 1.2223576154329283e-06, "loss": -0.0194, "reward": 0.1250000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 905 }, { "clip_ratio": 0.0, "completion_length": 581.8333435058594, "epoch": 0.604, "grad_norm": 0.6718376874923706, "kl": 0.6767578125, "learning_rate": 1.2189280281214128e-06, "loss": 0.1263, "reward": 0.3333333432674408, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 906 }, { "clip_ratio": 0.0, "completion_length": 678.5000152587891, "epoch": 0.6046666666666667, "grad_norm": 2.211164712905884, "kl": 1.3720703125, "learning_rate": 1.215499962932213e-06, "loss": 0.0667, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 907 }, { "clip_ratio": 0.0, "completion_length": 403.1666793823242, "epoch": 0.6053333333333333, "grad_norm": 0.7174047827720642, "kl": 0.2078857421875, "learning_rate": 1.2120734384297348e-06, "loss": -0.001, "reward": 0.4166666828095913, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.4166666828095913, "rewards/format_reward": 0.0, "step": 908 }, { "clip_ratio": 0.0, "completion_length": 640.8750305175781, "epoch": 0.606, "grad_norm": 0.3444939851760864, "kl": 0.988067626953125, "learning_rate": 1.2086484731700424e-06, "loss": 0.132, "reward": 0.1666666716337204, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 909 }, { "clip_ratio": 0.0, "completion_length": 604.7083435058594, "epoch": 0.6066666666666667, "grad_norm": 8.62305736541748, "kl": 1.6376953125, "learning_rate": 1.2052250857007548e-06, "loss": 0.2073, "reward": 0.2500000111758709, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.2500000111758709, "rewards/format_reward": 0.0, "step": 910 }, { "clip_ratio": 0.0, "completion_length": 422.2500114440918, "epoch": 0.6073333333333333, "grad_norm": 0.48178359866142273, "kl": 0.3240966796875, "learning_rate": 1.2018032945609486e-06, "loss": 0.0073, "reward": 0.5000000074505806, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.5000000074505806, "rewards/format_reward": 0.0, "step": 911 }, { "clip_ratio": 0.0, "completion_length": 398.4583435058594, "epoch": 0.608, "grad_norm": 0.4542020559310913, "kl": 0.4781494140625, "learning_rate": 1.1983831182810534e-06, "loss": 0.0282, "reward": 0.2083333432674408, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.2083333432674408, "rewards/format_reward": 0.0, "step": 912 }, { "clip_ratio": 0.0, "completion_length": 628.1666946411133, "epoch": 0.6086666666666667, "grad_norm": 0.5382109880447388, "kl": 1.17431640625, "learning_rate": 1.194964575382754e-06, "loss": 0.2014, "reward": 0.2083333395421505, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 913 }, { "clip_ratio": 0.0, "completion_length": 680.6250114440918, "epoch": 0.6093333333333333, "grad_norm": 3.9683516025543213, "kl": 2.43505859375, "learning_rate": 1.1915476843788921e-06, "loss": 0.0849, "reward": 0.2500000111758709, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.2500000111758709, "rewards/format_reward": 0.0, "step": 914 }, { "clip_ratio": 0.0, "completion_length": 569.9166946411133, "epoch": 0.61, "grad_norm": 0.6038851141929626, "kl": 0.9873046875, "learning_rate": 1.1881324637733612e-06, "loss": 0.092, "reward": 0.2083333432674408, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.2083333432674408, "rewards/format_reward": 0.0, "step": 915 }, { "clip_ratio": 0.0, "completion_length": 485.8333435058594, "epoch": 0.6106666666666667, "grad_norm": 0.418722540140152, "kl": 0.61083984375, "learning_rate": 1.1847189320610094e-06, "loss": 0.0307, "reward": 0.25, "reward_std": 0.273861289024353, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "step": 916 }, { "clip_ratio": 0.0, "completion_length": 561.7083587646484, "epoch": 0.6113333333333333, "grad_norm": 0.6089925765991211, "kl": 0.9599609375, "learning_rate": 1.1813071077275383e-06, "loss": 0.0898, "reward": 0.3333333432674408, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 917 }, { "clip_ratio": 0.0, "completion_length": 472.20835876464844, "epoch": 0.612, "grad_norm": 0.6968267560005188, "kl": 0.5845947265625, "learning_rate": 1.1778970092494053e-06, "loss": 0.0233, "reward": 0.2083333358168602, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.2083333358168602, "rewards/format_reward": 0.0, "step": 918 }, { "clip_ratio": 0.0, "completion_length": 591.6666870117188, "epoch": 0.6126666666666667, "grad_norm": 0.7926375865936279, "kl": 0.961669921875, "learning_rate": 1.17448865509372e-06, "loss": 0.0435, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 919 }, { "clip_ratio": 0.0, "completion_length": 800.9167022705078, "epoch": 0.6133333333333333, "grad_norm": 0.3270564079284668, "kl": 1.31396484375, "learning_rate": 1.1710820637181448e-06, "loss": 0.0911, "reward": 0.1250000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 920 }, { "clip_ratio": 0.0, "completion_length": 621.2083435058594, "epoch": 0.614, "grad_norm": 1.0346511602401733, "kl": 1.0712890625, "learning_rate": 1.1676772535707992e-06, "loss": 0.1442, "reward": 0.4583333432674408, "reward_std": 0.37592336535453796, "rewards/accuracy_reward": 0.4583333432674408, "rewards/format_reward": 0.0, "step": 921 }, { "clip_ratio": 0.0, "completion_length": 583.6666793823242, "epoch": 0.6146666666666667, "grad_norm": 0.8383022546768188, "kl": 0.796142578125, "learning_rate": 1.164274243090153e-06, "loss": 0.1638, "reward": 0.4166666828095913, "reward_std": 0.3332236111164093, "rewards/accuracy_reward": 0.4166666828095913, "rewards/format_reward": 0.0, "step": 922 }, { "clip_ratio": 0.0, "completion_length": 624.6666870117188, "epoch": 0.6153333333333333, "grad_norm": 0.508921205997467, "kl": 1.0576171875, "learning_rate": 1.1608730507049326e-06, "loss": 0.1578, "reward": 0.2500000037252903, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.0, "step": 923 }, { "clip_ratio": 0.0, "completion_length": 536.5416831970215, "epoch": 0.616, "grad_norm": 1.0375511646270752, "kl": 0.459228515625, "learning_rate": 1.1574736948340164e-06, "loss": 0.1289, "reward": 0.4583333432674408, "reward_std": 0.37592335790395737, "rewards/accuracy_reward": 0.4583333432674408, "rewards/format_reward": 0.0, "step": 924 }, { "clip_ratio": 0.0, "completion_length": 518.5833587646484, "epoch": 0.6166666666666667, "grad_norm": 0.30449292063713074, "kl": 0.6253662109375, "learning_rate": 1.1540761938863398e-06, "loss": 0.0902, "reward": 0.0833333358168602, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 925 }, { "clip_ratio": 0.0, "completion_length": 511.2083435058594, "epoch": 0.6173333333333333, "grad_norm": 0.37058714032173157, "kl": 0.686279296875, "learning_rate": 1.1506805662607914e-06, "loss": 0.0721, "reward": 0.2500000037252903, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.0, "step": 926 }, { "clip_ratio": 0.0, "completion_length": 593.9166793823242, "epoch": 0.618, "grad_norm": 0.5731404423713684, "kl": 0.59771728515625, "learning_rate": 1.147286830346115e-06, "loss": 0.0253, "reward": 0.0833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 927 }, { "clip_ratio": 0.0, "completion_length": 508.75001525878906, "epoch": 0.6186666666666667, "grad_norm": 0.4166342616081238, "kl": 0.4427490234375, "learning_rate": 1.1438950045208108e-06, "loss": 0.1151, "reward": 0.5833333432674408, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.5833333432674408, "rewards/format_reward": 0.0, "step": 928 }, { "clip_ratio": 0.0, "completion_length": 563.5000190734863, "epoch": 0.6193333333333333, "grad_norm": 0.345920205116272, "kl": 1.26953125, "learning_rate": 1.1405051071530346e-06, "loss": 0.226, "reward": 0.1666666716337204, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 929 }, { "clip_ratio": 0.0, "completion_length": 539.5833587646484, "epoch": 0.62, "grad_norm": 0.6544208526611328, "kl": 0.891357421875, "learning_rate": 1.1371171566004986e-06, "loss": 0.0689, "reward": 0.4166666716337204, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.4166666716337204, "rewards/format_reward": 0.0, "step": 930 }, { "clip_ratio": 0.0, "completion_length": 491.29168701171875, "epoch": 0.6206666666666667, "grad_norm": 0.6895833015441895, "kl": 1.15283203125, "learning_rate": 1.1337311712103722e-06, "loss": 0.0411, "reward": 0.2916666716337204, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.2916666716337204, "rewards/format_reward": 0.0, "step": 931 }, { "clip_ratio": 0.0, "completion_length": 656.4166717529297, "epoch": 0.6213333333333333, "grad_norm": 0.46006256341934204, "kl": 0.46832275390625, "learning_rate": 1.1303471693191835e-06, "loss": 0.072, "reward": 0.1666666716337204, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 932 }, { "clip_ratio": 0.0, "completion_length": 533.5416793823242, "epoch": 0.622, "grad_norm": 0.3755480945110321, "kl": 1.186767578125, "learning_rate": 1.1269651692527181e-06, "loss": 0.1097, "reward": 0.3333333469927311, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.3333333469927311, "rewards/format_reward": 0.0, "step": 933 }, { "clip_ratio": 0.0, "completion_length": 564.4166793823242, "epoch": 0.6226666666666667, "grad_norm": 0.3518484830856323, "kl": 0.676513671875, "learning_rate": 1.1235851893259208e-06, "loss": 0.0864, "reward": 0.1250000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 934 }, { "clip_ratio": 0.0, "completion_length": 517.2916870117188, "epoch": 0.6233333333333333, "grad_norm": 0.3544982373714447, "kl": 0.2562255859375, "learning_rate": 1.1202072478427976e-06, "loss": 0.1224, "reward": 0.7500000149011612, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.7500000149011612, "rewards/format_reward": 0.0, "step": 935 }, { "clip_ratio": 0.0, "completion_length": 440.5000114440918, "epoch": 0.624, "grad_norm": 0.3357725441455841, "kl": 0.59716796875, "learning_rate": 1.1168313630963144e-06, "loss": 0.0319, "reward": 0.4166666716337204, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.4166666716337204, "rewards/format_reward": 0.0, "step": 936 }, { "clip_ratio": 0.0, "completion_length": 532.5000228881836, "epoch": 0.6246666666666667, "grad_norm": 0.4264965057373047, "kl": 0.449462890625, "learning_rate": 1.1134575533682993e-06, "loss": 0.0715, "reward": 0.5416666865348816, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.5416666865348816, "rewards/format_reward": 0.0, "step": 937 }, { "clip_ratio": 0.0, "completion_length": 463.7083435058594, "epoch": 0.6253333333333333, "grad_norm": 0.2601344883441925, "kl": 0.3172607421875, "learning_rate": 1.110085836929344e-06, "loss": 0.0113, "reward": 0.2083333432674408, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.2083333432674408, "rewards/format_reward": 0.0, "step": 938 }, { "clip_ratio": 0.0, "completion_length": 492.7500228881836, "epoch": 0.626, "grad_norm": 0.8574249744415283, "kl": 0.753143310546875, "learning_rate": 1.1067162320387032e-06, "loss": 0.0297, "reward": 0.5, "reward_std": 0.0, "rewards/accuracy_reward": 0.5, "rewards/format_reward": 0.0, "step": 939 }, { "clip_ratio": 0.0, "completion_length": 507.2083435058594, "epoch": 0.6266666666666667, "grad_norm": 0.8836904764175415, "kl": 0.719482421875, "learning_rate": 1.103348756944197e-06, "loss": 0.0124, "reward": 0.3750000111758709, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 940 }, { "clip_ratio": 0.0, "completion_length": 283.37500762939453, "epoch": 0.6273333333333333, "grad_norm": 0.26742827892303467, "kl": 0.09991455078125, "learning_rate": 1.0999834298821117e-06, "loss": 0.0638, "reward": 0.7500000074505806, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.7500000074505806, "rewards/format_reward": 0.0, "step": 941 }, { "clip_ratio": 0.0, "completion_length": 460.7083435058594, "epoch": 0.628, "grad_norm": 0.7111197710037231, "kl": 0.45458984375, "learning_rate": 1.0966202690771015e-06, "loss": 0.0352, "reward": 0.2916666753590107, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.2916666753590107, "rewards/format_reward": 0.0, "step": 942 }, { "clip_ratio": 0.0, "completion_length": 642.0416793823242, "epoch": 0.6286666666666667, "grad_norm": 0.29692885279655457, "kl": 0.37060546875, "learning_rate": 1.0932592927420893e-06, "loss": 0.0642, "reward": 0.2916666753590107, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.2916666753590107, "rewards/format_reward": 0.0, "step": 943 }, { "clip_ratio": 0.0, "completion_length": 517.8333511352539, "epoch": 0.6293333333333333, "grad_norm": 0.3846733570098877, "kl": 0.399658203125, "learning_rate": 1.0899005190781674e-06, "loss": 0.021, "reward": 0.25, "reward_std": 0.0, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "step": 944 }, { "clip_ratio": 0.0, "completion_length": 333.9583435058594, "epoch": 0.63, "grad_norm": 0.983694314956665, "kl": 0.263671875, "learning_rate": 1.0865439662745013e-06, "loss": 0.1027, "reward": 0.708333358168602, "reward_std": 0.4893604442477226, "rewards/accuracy_reward": 0.708333358168602, "rewards/format_reward": 0.0, "step": 945 }, { "clip_ratio": 0.0, "completion_length": 430.2500228881836, "epoch": 0.6306666666666667, "grad_norm": 0.5921170711517334, "kl": 0.2559814453125, "learning_rate": 1.0831896525082282e-06, "loss": 0.0971, "reward": 0.2500000074505806, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 946 }, { "clip_ratio": 0.0, "completion_length": 649.0000228881836, "epoch": 0.6313333333333333, "grad_norm": 1.594773292541504, "kl": 0.823974609375, "learning_rate": 1.0798375959443601e-06, "loss": 0.2027, "reward": 0.4583333469927311, "reward_std": 0.4701542556285858, "rewards/accuracy_reward": 0.4583333469927311, "rewards/format_reward": 0.0, "step": 947 }, { "clip_ratio": 0.0, "completion_length": 568.5833435058594, "epoch": 0.632, "grad_norm": 0.6104963421821594, "kl": 0.61273193359375, "learning_rate": 1.0764878147356852e-06, "loss": 0.0268, "reward": 0.0833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 948 }, { "clip_ratio": 0.0, "completion_length": 535.5000076293945, "epoch": 0.6326666666666667, "grad_norm": 0.37660136818885803, "kl": 0.285888671875, "learning_rate": 1.0731403270226717e-06, "loss": 0.009, "reward": 0.3333333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 949 }, { "clip_ratio": 0.0, "completion_length": 551.2083587646484, "epoch": 0.6333333333333333, "grad_norm": 0.1579110473394394, "kl": 0.2252197265625, "learning_rate": 1.069795150933365e-06, "loss": 0.0403, "reward": 0.0416666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 950 }, { "clip_ratio": 0.0, "completion_length": 769.9583587646484, "epoch": 0.634, "grad_norm": 1.0768749713897705, "kl": 1.2216796875, "learning_rate": 1.0664523045832924e-06, "loss": 0.067, "reward": 0.125, "reward_std": 0.1369306445121765, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "step": 951 }, { "clip_ratio": 0.0, "completion_length": 382.62500762939453, "epoch": 0.6346666666666667, "grad_norm": 0.3160300850868225, "kl": 0.27606201171875, "learning_rate": 1.0631118060753668e-06, "loss": 0.0373, "reward": 0.1666666679084301, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 952 }, { "clip_ratio": 0.0, "completion_length": 671.9583587646484, "epoch": 0.6353333333333333, "grad_norm": 1.1302930116653442, "kl": 0.21875, "learning_rate": 1.0597736734997847e-06, "loss": 0.042, "reward": 0.4583333432674408, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.4583333432674408, "rewards/format_reward": 0.0, "step": 953 }, { "clip_ratio": 0.0, "completion_length": 449.0416717529297, "epoch": 0.636, "grad_norm": 0.46357718110084534, "kl": 0.16497802734375, "learning_rate": 1.0564379249339306e-06, "loss": -0.0444, "reward": 0.3333333395421505, "reward_std": 0.3680921792984009, "rewards/accuracy_reward": 0.3333333395421505, "rewards/format_reward": 0.0, "step": 954 }, { "clip_ratio": 0.0, "completion_length": 566.5000076293945, "epoch": 0.6366666666666667, "grad_norm": 0.5274302959442139, "kl": 0.26068115234375, "learning_rate": 1.053104578442277e-06, "loss": 0.0924, "reward": 0.2083333395421505, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 955 }, { "clip_ratio": 0.0, "completion_length": 331.12500381469727, "epoch": 0.6373333333333333, "grad_norm": 0.22888807952404022, "kl": 0.2939453125, "learning_rate": 1.0497736520762914e-06, "loss": 0.0152, "reward": 0.5416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.5416666679084301, "rewards/format_reward": 0.0, "step": 956 }, { "clip_ratio": 0.0, "completion_length": 529.0416717529297, "epoch": 0.638, "grad_norm": 0.3116917908191681, "kl": 0.17706298828125, "learning_rate": 1.0464451638743333e-06, "loss": 0.0557, "reward": 0.2916666753590107, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.2916666753590107, "rewards/format_reward": 0.0, "step": 957 }, { "clip_ratio": 0.0, "completion_length": 538.0416870117188, "epoch": 0.6386666666666667, "grad_norm": 0.2928798496723175, "kl": 0.36151123046875, "learning_rate": 1.0431191318615581e-06, "loss": 0.0061, "reward": 0.1250000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 958 }, { "clip_ratio": 0.0, "completion_length": 486.5416793823242, "epoch": 0.6393333333333333, "grad_norm": 0.2933100759983063, "kl": 0.4166259765625, "learning_rate": 1.0397955740498216e-06, "loss": 0.0494, "reward": 0.2500000074505806, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 959 }, { "clip_ratio": 0.0, "completion_length": 476.50000762939453, "epoch": 0.64, "grad_norm": 0.1371784806251526, "kl": 0.091217041015625, "learning_rate": 1.036474508437579e-06, "loss": 0.0051, "reward": 0.2083333432674408, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.2083333432674408, "rewards/format_reward": 0.0, "step": 960 }, { "clip_ratio": 0.0, "completion_length": 466.2916831970215, "epoch": 0.6406666666666667, "grad_norm": 0.2850760221481323, "kl": 0.18048095703125, "learning_rate": 1.0331559530097903e-06, "loss": 0.0553, "reward": 0.4166666716337204, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.4166666716337204, "rewards/format_reward": 0.0, "step": 961 }, { "clip_ratio": 0.0, "completion_length": 706.2916870117188, "epoch": 0.6413333333333333, "grad_norm": 0.9765766859054565, "kl": 0.82421875, "learning_rate": 1.0298399257378207e-06, "loss": 0.2558, "reward": 0.2916666679084301, "reward_std": 0.37592336535453796, "rewards/accuracy_reward": 0.2916666679084301, "rewards/format_reward": 0.0, "step": 962 }, { "clip_ratio": 0.0, "completion_length": 344.79168701171875, "epoch": 0.642, "grad_norm": 0.18813279271125793, "kl": 0.4063720703125, "learning_rate": 1.0265264445793464e-06, "loss": 0.0861, "reward": 0.5833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.5833333358168602, "rewards/format_reward": 0.0, "step": 963 }, { "clip_ratio": 0.0, "completion_length": 475.62501525878906, "epoch": 0.6426666666666667, "grad_norm": 0.3587769567966461, "kl": 0.62725830078125, "learning_rate": 1.0232155274782533e-06, "loss": 0.0048, "reward": 0.5833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.5833333358168602, "rewards/format_reward": 0.0, "step": 964 }, { "clip_ratio": 0.0, "completion_length": 453.62501525878906, "epoch": 0.6433333333333333, "grad_norm": 0.3768203854560852, "kl": 0.3992919921875, "learning_rate": 1.0199071923645425e-06, "loss": 0.0644, "reward": 0.5000000074505806, "reward_std": 0.3872983753681183, "rewards/accuracy_reward": 0.5000000074505806, "rewards/format_reward": 0.0, "step": 965 }, { "clip_ratio": 0.0, "completion_length": 328.8750114440918, "epoch": 0.644, "grad_norm": 1.1605732440948486, "kl": 0.215576171875, "learning_rate": 1.0166014571542335e-06, "loss": 0.1018, "reward": 0.9166666865348816, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.9166666865348816, "rewards/format_reward": 0.0, "step": 966 }, { "clip_ratio": 0.0, "completion_length": 626.3750228881836, "epoch": 0.6446666666666667, "grad_norm": 1.275178074836731, "kl": 0.8248291015625, "learning_rate": 1.0132983397492647e-06, "loss": 0.0912, "reward": 0.4583333507180214, "reward_std": 0.4893604516983032, "rewards/accuracy_reward": 0.4583333507180214, "rewards/format_reward": 0.0, "step": 967 }, { "clip_ratio": 0.0, "completion_length": 538.1250152587891, "epoch": 0.6453333333333333, "grad_norm": 0.43228450417518616, "kl": 0.8701171875, "learning_rate": 1.0099978580373988e-06, "loss": 0.1123, "reward": 0.1666666679084301, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 968 }, { "clip_ratio": 0.0, "completion_length": 535.7916831970215, "epoch": 0.646, "grad_norm": 0.9224567413330078, "kl": 1.173095703125, "learning_rate": 1.0067000298921252e-06, "loss": 0.0994, "reward": 0.2500000111758709, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.2500000111758709, "rewards/format_reward": 0.0, "step": 969 }, { "clip_ratio": 0.0, "completion_length": 491.41667556762695, "epoch": 0.6466666666666666, "grad_norm": 1.39867103099823, "kl": 1.29779052734375, "learning_rate": 1.003404873172563e-06, "loss": 0.1227, "reward": 0.2916666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.2916666679084301, "rewards/format_reward": 0.0, "step": 970 }, { "clip_ratio": 0.0, "completion_length": 477.3750228881836, "epoch": 0.6473333333333333, "grad_norm": 0.43235746026039124, "kl": 0.5018310546875, "learning_rate": 1.0001124057233644e-06, "loss": 0.042, "reward": 0.2916666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.2916666679084301, "rewards/format_reward": 0.0, "step": 971 }, { "clip_ratio": 0.0, "completion_length": 542.9166793823242, "epoch": 0.648, "grad_norm": 0.3811420202255249, "kl": 0.2989501953125, "learning_rate": 9.968226453746177e-07, "loss": 0.0561, "reward": 0.1666666716337204, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 972 }, { "clip_ratio": 0.0, "completion_length": 762.7500152587891, "epoch": 0.6486666666666666, "grad_norm": 1.090301513671875, "kl": 1.144775390625, "learning_rate": 9.935356099417518e-07, "loss": 0.0554, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 973 }, { "clip_ratio": 0.0, "completion_length": 604.2500076293945, "epoch": 0.6493333333333333, "grad_norm": 0.5233350396156311, "kl": 1.17919921875, "learning_rate": 9.902513172254386e-07, "loss": 0.0938, "reward": 0.2916666716337204, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.2916666716337204, "rewards/format_reward": 0.0, "step": 974 }, { "clip_ratio": 0.0, "completion_length": 543.1666717529297, "epoch": 0.65, "grad_norm": 0.875827431678772, "kl": 0.7218017578125, "learning_rate": 9.86969785011497e-07, "loss": 0.1698, "reward": 0.2916666753590107, "reward_std": 0.3602609932422638, "rewards/accuracy_reward": 0.2916666753590107, "rewards/format_reward": 0.0, "step": 975 }, { "clip_ratio": 0.0, "completion_length": 520.0833511352539, "epoch": 0.6506666666666666, "grad_norm": 1.0364644527435303, "kl": 0.653076171875, "learning_rate": 9.836910310707973e-07, "loss": 0.1495, "reward": 0.5000000111758709, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.5000000111758709, "rewards/format_reward": 0.0, "step": 976 }, { "clip_ratio": 0.0, "completion_length": 640.1666793823242, "epoch": 0.6513333333333333, "grad_norm": 0.3909541070461273, "kl": 0.63232421875, "learning_rate": 9.804150731591631e-07, "loss": 0.1615, "reward": 0.125, "reward_std": 0.1369306445121765, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "step": 977 }, { "clip_ratio": 0.0, "completion_length": 620.6666831970215, "epoch": 0.652, "grad_norm": 1.425905704498291, "kl": 0.6329345703125, "learning_rate": 9.771419290172774e-07, "loss": 0.0614, "reward": 0.3333333469927311, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.3333333469927311, "rewards/format_reward": 0.0, "step": 978 }, { "clip_ratio": 0.0, "completion_length": 696.8750305175781, "epoch": 0.6526666666666666, "grad_norm": 0.8144387006759644, "kl": 0.98095703125, "learning_rate": 9.738716163705848e-07, "loss": 0.0414, "reward": 0.0416666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 979 }, { "clip_ratio": 0.0, "completion_length": 613.6666870117188, "epoch": 0.6533333333333333, "grad_norm": 0.2977432608604431, "kl": 0.700439453125, "learning_rate": 9.70604152929197e-07, "loss": 0.092, "reward": 0.1666666716337204, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 980 }, { "clip_ratio": 0.0, "completion_length": 489.8750228881836, "epoch": 0.654, "grad_norm": 0.2399890422821045, "kl": 0.312255859375, "learning_rate": 9.673395563877947e-07, "loss": -0.0202, "reward": 0.5416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.5416666679084301, "rewards/format_reward": 0.0, "step": 981 }, { "clip_ratio": 0.0, "completion_length": 538.7083435058594, "epoch": 0.6546666666666666, "grad_norm": 0.3180677890777588, "kl": 0.589599609375, "learning_rate": 9.64077844425534e-07, "loss": 0.03, "reward": 0.0416666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 982 }, { "clip_ratio": 0.0, "completion_length": 372.00000762939453, "epoch": 0.6553333333333333, "grad_norm": 0.4150152802467346, "kl": 0.410888671875, "learning_rate": 9.608190347059505e-07, "loss": 0.0864, "reward": 0.3333333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 983 }, { "clip_ratio": 0.0, "completion_length": 671.6666793823242, "epoch": 0.656, "grad_norm": 0.3171081840991974, "kl": 0.56787109375, "learning_rate": 9.575631448768617e-07, "loss": 0.0335, "reward": 0.0833333358168602, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 984 }, { "clip_ratio": 0.0, "completion_length": 671.1250152587891, "epoch": 0.6566666666666666, "grad_norm": 0.4894086420536041, "kl": 0.44775390625, "learning_rate": 9.543101925702732e-07, "loss": 0.0598, "reward": 0.3750000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.3750000037252903, "rewards/format_reward": 0.0, "step": 985 }, { "clip_ratio": 0.0, "completion_length": 420.87501525878906, "epoch": 0.6573333333333333, "grad_norm": 1.7718946933746338, "kl": 0.3236083984375, "learning_rate": 9.510601954022818e-07, "loss": 0.2001, "reward": 0.4166666828095913, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.4166666828095913, "rewards/format_reward": 0.0, "step": 986 }, { "clip_ratio": 0.0, "completion_length": 434.9166793823242, "epoch": 0.658, "grad_norm": 1.9551725387573242, "kl": 0.8636474609375, "learning_rate": 9.478131709729831e-07, "loss": 0.1698, "reward": 0.666666679084301, "reward_std": 0.3680921792984009, "rewards/accuracy_reward": 0.666666679084301, "rewards/format_reward": 0.0, "step": 987 }, { "clip_ratio": 0.0, "completion_length": 321.1666793823242, "epoch": 0.6586666666666666, "grad_norm": 1.1909085512161255, "kl": 0.4844970703125, "learning_rate": 9.445691368663725e-07, "loss": 0.0143, "reward": 0.5833333544433117, "reward_std": 0.4431168660521507, "rewards/accuracy_reward": 0.5833333544433117, "rewards/format_reward": 0.0, "step": 988 }, { "clip_ratio": 0.0, "completion_length": 608.0416793823242, "epoch": 0.6593333333333333, "grad_norm": 0.5221753716468811, "kl": 1.109375, "learning_rate": 9.413281106502507e-07, "loss": 0.2684, "reward": 0.3750000074505806, "reward_std": 0.39512956142425537, "rewards/accuracy_reward": 0.3750000074505806, "rewards/format_reward": 0.0, "step": 989 }, { "clip_ratio": 0.0, "completion_length": 523.2916717529297, "epoch": 0.66, "grad_norm": 1.7951878309249878, "kl": 1.185546875, "learning_rate": 9.380901098761319e-07, "loss": 0.2806, "reward": 0.416666679084301, "reward_std": 0.3872983753681183, "rewards/accuracy_reward": 0.416666679084301, "rewards/format_reward": 0.0, "step": 990 }, { "clip_ratio": 0.0, "completion_length": 583.7083435058594, "epoch": 0.6606666666666666, "grad_norm": 0.42406386137008667, "kl": 0.5498046875, "learning_rate": 9.348551520791446e-07, "loss": 0.0654, "reward": 0.125, "reward_std": 0.1369306445121765, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "step": 991 }, { "clip_ratio": 0.0, "completion_length": 514.458366394043, "epoch": 0.6613333333333333, "grad_norm": 0.5236000418663025, "kl": 1.4619140625, "learning_rate": 9.316232547779386e-07, "loss": 0.1676, "reward": 0.5833333395421505, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.5833333395421505, "rewards/format_reward": 0.0, "step": 992 }, { "clip_ratio": 0.0, "completion_length": 598.5416793823242, "epoch": 0.662, "grad_norm": 0.4790690839290619, "kl": 1.7197265625, "learning_rate": 9.283944354745889e-07, "loss": 0.2909, "reward": 0.3333333432674408, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 993 }, { "clip_ratio": 0.0, "completion_length": 558.5416831970215, "epoch": 0.6626666666666666, "grad_norm": 0.43485376238822937, "kl": 0.9000244140625, "learning_rate": 9.251687116545041e-07, "loss": 0.1771, "reward": 0.4583333469927311, "reward_std": 0.4701542407274246, "rewards/accuracy_reward": 0.4583333469927311, "rewards/format_reward": 0.0, "step": 994 }, { "clip_ratio": 0.0, "completion_length": 631.8333511352539, "epoch": 0.6633333333333333, "grad_norm": 0.8245652318000793, "kl": 1.542236328125, "learning_rate": 9.219461007863277e-07, "loss": 0.1727, "reward": 0.291666679084301, "reward_std": 0.3061862140893936, "rewards/accuracy_reward": 0.291666679084301, "rewards/format_reward": 0.0, "step": 995 }, { "clip_ratio": 0.0, "completion_length": 535.7500152587891, "epoch": 0.664, "grad_norm": 0.39318689703941345, "kl": 1.05712890625, "learning_rate": 9.187266203218456e-07, "loss": 0.13, "reward": 0.2500000074505806, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 996 }, { "clip_ratio": 0.0, "completion_length": 615.4583587646484, "epoch": 0.6646666666666666, "grad_norm": 1.2120145559310913, "kl": 1.8477783203125, "learning_rate": 9.155102876958917e-07, "loss": 0.1375, "reward": 0.2083333432674408, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.2083333432674408, "rewards/format_reward": 0.0, "step": 997 }, { "clip_ratio": 0.0, "completion_length": 618.2083435058594, "epoch": 0.6653333333333333, "grad_norm": 0.7425616383552551, "kl": 1.82177734375, "learning_rate": 9.122971203262524e-07, "loss": 0.1433, "reward": 0.2500000074505806, "reward_std": 0.3872983753681183, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 998 }, { "clip_ratio": 0.0, "completion_length": 625.0833358764648, "epoch": 0.666, "grad_norm": 1.9747779369354248, "kl": 2.0625, "learning_rate": 9.090871356135733e-07, "loss": 0.1641, "reward": 0.0833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 999 }, { "clip_ratio": 0.0, "completion_length": 577.3333435058594, "epoch": 0.6666666666666666, "grad_norm": 0.6043461561203003, "kl": 0.91656494140625, "learning_rate": 9.058803509412648e-07, "loss": 0.0288, "reward": 0.0833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 1000 }, { "clip_ratio": 0.0, "completion_length": 479.75001525878906, "epoch": 0.6673333333333333, "grad_norm": 1.0055632591247559, "kl": 0.89483642578125, "learning_rate": 9.026767836754074e-07, "loss": 0.1137, "reward": 0.4583333432674408, "reward_std": 0.39512956142425537, "rewards/accuracy_reward": 0.4583333432674408, "rewards/format_reward": 0.0, "step": 1001 }, { "clip_ratio": 0.0, "completion_length": 726.0833587646484, "epoch": 0.668, "grad_norm": 2.04168438911438, "kl": 2.061279296875, "learning_rate": 8.994764511646579e-07, "loss": 0.0845, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 1002 }, { "clip_ratio": 0.0, "completion_length": 626.6666793823242, "epoch": 0.6686666666666666, "grad_norm": 1.3532440662384033, "kl": 1.44012451171875, "learning_rate": 8.962793707401554e-07, "loss": 0.044, "reward": 0.375, "reward_std": 0.1369306445121765, "rewards/accuracy_reward": 0.375, "rewards/format_reward": 0.0, "step": 1003 }, { "clip_ratio": 0.0, "completion_length": 582.3333511352539, "epoch": 0.6693333333333333, "grad_norm": 1.9247932434082031, "kl": 0.70489501953125, "learning_rate": 8.930855597154285e-07, "loss": 0.0814, "reward": 0.1666666679084301, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 1004 }, { "clip_ratio": 0.0, "completion_length": 551.9166831970215, "epoch": 0.67, "grad_norm": 0.6744492650032043, "kl": 0.47686767578125, "learning_rate": 8.898950353863e-07, "loss": -0.0268, "reward": 0.2500000111758709, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.2500000111758709, "rewards/format_reward": 0.0, "step": 1005 }, { "clip_ratio": 0.0, "completion_length": 714.5000152587891, "epoch": 0.6706666666666666, "grad_norm": 0.30378973484039307, "kl": 0.427490234375, "learning_rate": 8.867078150307931e-07, "loss": 0.0666, "reward": 0.0833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 1006 }, { "clip_ratio": 0.0, "completion_length": 644.5416946411133, "epoch": 0.6713333333333333, "grad_norm": 0.326728492975235, "kl": 0.867431640625, "learning_rate": 8.835239159090407e-07, "loss": 0.0436, "reward": 0.291666679084301, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.291666679084301, "rewards/format_reward": 0.0, "step": 1007 }, { "clip_ratio": 0.0, "completion_length": 463.0833511352539, "epoch": 0.672, "grad_norm": 0.369308203458786, "kl": 0.28070068359375, "learning_rate": 8.803433552631875e-07, "loss": 0.019, "reward": 0.4166666716337204, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.4166666716337204, "rewards/format_reward": 0.0, "step": 1008 }, { "clip_ratio": 0.0, "completion_length": 682.2500152587891, "epoch": 0.6726666666666666, "grad_norm": 0.24464409053325653, "kl": 0.29229736328125, "learning_rate": 8.771661503173005e-07, "loss": 0.0205, "reward": 0.0833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 1009 }, { "clip_ratio": 0.0, "completion_length": 548.4583435058594, "epoch": 0.6733333333333333, "grad_norm": 0.5054168105125427, "kl": 0.3243408203125, "learning_rate": 8.739923182772733e-07, "loss": 0.037, "reward": 0.1250000037252903, "reward_std": 0.3061862215399742, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 1010 }, { "clip_ratio": 0.0, "completion_length": 654.0416717529297, "epoch": 0.674, "grad_norm": 1.1505740880966187, "kl": 0.38232421875, "learning_rate": 8.708218763307343e-07, "loss": 0.1709, "reward": 0.2500000037252903, "reward_std": 0.44311685860157013, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.0, "step": 1011 }, { "clip_ratio": 0.0, "completion_length": 635.8750152587891, "epoch": 0.6746666666666666, "grad_norm": 1.3524935245513916, "kl": 0.396484375, "learning_rate": 8.676548416469538e-07, "loss": 0.0808, "reward": 0.4583333544433117, "reward_std": 0.3061862140893936, "rewards/accuracy_reward": 0.4583333544433117, "rewards/format_reward": 0.0, "step": 1012 }, { "clip_ratio": 0.0, "completion_length": 487.2500228881836, "epoch": 0.6753333333333333, "grad_norm": 0.37081387639045715, "kl": 0.3470458984375, "learning_rate": 8.644912313767479e-07, "loss": 0.0842, "reward": 0.4583333395421505, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.4583333395421505, "rewards/format_reward": 0.0, "step": 1013 }, { "clip_ratio": 0.0, "completion_length": 564.6250076293945, "epoch": 0.676, "grad_norm": 0.7928094267845154, "kl": 0.3446044921875, "learning_rate": 8.613310626523911e-07, "loss": -0.0121, "reward": 0.4166666753590107, "reward_std": 0.4971916377544403, "rewards/accuracy_reward": 0.4166666753590107, "rewards/format_reward": 0.0, "step": 1014 }, { "clip_ratio": 0.0, "completion_length": 392.5000190734863, "epoch": 0.6766666666666666, "grad_norm": 0.17947423458099365, "kl": 0.43426513671875, "learning_rate": 8.58174352587519e-07, "loss": 0.0436, "reward": 0.3333333358168602, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 1015 }, { "clip_ratio": 0.0, "completion_length": 458.1250114440918, "epoch": 0.6773333333333333, "grad_norm": 0.2064787894487381, "kl": 0.143798828125, "learning_rate": 8.550211182770362e-07, "loss": -0.0302, "reward": 0.375, "reward_std": 0.1369306445121765, "rewards/accuracy_reward": 0.375, "rewards/format_reward": 0.0, "step": 1016 }, { "clip_ratio": 0.0, "completion_length": 498.79168701171875, "epoch": 0.678, "grad_norm": 0.24779094755649567, "kl": 0.281158447265625, "learning_rate": 8.518713767970263e-07, "loss": 0.011, "reward": 0.375, "reward_std": 0.1369306445121765, "rewards/accuracy_reward": 0.375, "rewards/format_reward": 0.0, "step": 1017 }, { "clip_ratio": 0.0, "completion_length": 535.0833587646484, "epoch": 0.6786666666666666, "grad_norm": 0.25123095512390137, "kl": 0.256591796875, "learning_rate": 8.48725145204658e-07, "loss": 0.0116, "reward": 0.4583333432674408, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.4583333432674408, "rewards/format_reward": 0.0, "step": 1018 }, { "clip_ratio": 0.0, "completion_length": 496.5833396911621, "epoch": 0.6793333333333333, "grad_norm": 0.352893203496933, "kl": 0.57781982421875, "learning_rate": 8.4558244053809e-07, "loss": 0.0551, "reward": 0.5833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.5833333358168602, "rewards/format_reward": 0.0, "step": 1019 }, { "clip_ratio": 0.0, "completion_length": 496.8333396911621, "epoch": 0.68, "grad_norm": 1.4178426265716553, "kl": 0.347900390625, "learning_rate": 8.424432798163837e-07, "loss": 0.0688, "reward": 0.4166666865348816, "reward_std": 0.20412415266036987, "rewards/accuracy_reward": 0.4166666865348816, "rewards/format_reward": 0.0, "step": 1020 }, { "clip_ratio": 0.0, "completion_length": 463.2083511352539, "epoch": 0.6806666666666666, "grad_norm": 0.03827522695064545, "kl": 0.1883544921875, "learning_rate": 8.393076800394081e-07, "loss": 0.0036, "reward": 0.25, "reward_std": 0.0, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "step": 1021 }, { "clip_ratio": 0.0, "completion_length": 554.3750228881836, "epoch": 0.6813333333333333, "grad_norm": 0.6928570866584778, "kl": 0.719482421875, "learning_rate": 8.361756581877467e-07, "loss": 0.154, "reward": 0.2083333395421505, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 1022 }, { "clip_ratio": 0.0, "completion_length": 384.0833511352539, "epoch": 0.682, "grad_norm": 0.17225761711597443, "kl": 0.1800537109375, "learning_rate": 8.330472312226092e-07, "loss": 0.0276, "reward": 0.2083333432674408, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.2083333432674408, "rewards/format_reward": 0.0, "step": 1023 }, { "clip_ratio": 0.0, "completion_length": 491.8750228881836, "epoch": 0.6826666666666666, "grad_norm": 0.6878138184547424, "kl": 0.37158203125, "learning_rate": 8.299224160857353e-07, "loss": 0.0352, "reward": 0.416666679084301, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.416666679084301, "rewards/format_reward": 0.0, "step": 1024 }, { "clip_ratio": 0.0, "completion_length": 563.4583511352539, "epoch": 0.6833333333333333, "grad_norm": 0.17288467288017273, "kl": 0.5216064453125, "learning_rate": 8.268012296993068e-07, "loss": 0.0378, "reward": 0.0833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 1025 }, { "clip_ratio": 0.0, "completion_length": 362.4583435058594, "epoch": 0.684, "grad_norm": 0.8998779654502869, "kl": 0.52374267578125, "learning_rate": 8.236836889658543e-07, "loss": 0.0096, "reward": 0.6250000149011612, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.6250000149011612, "rewards/format_reward": 0.0, "step": 1026 }, { "clip_ratio": 0.0, "completion_length": 511.4583435058594, "epoch": 0.6846666666666666, "grad_norm": 0.289468914270401, "kl": 0.3463134765625, "learning_rate": 8.205698107681638e-07, "loss": -0.0137, "reward": 0.1666666679084301, "reward_std": 0.23899272084236145, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 1027 }, { "clip_ratio": 0.0, "completion_length": 490.66668701171875, "epoch": 0.6853333333333333, "grad_norm": 0.6873383522033691, "kl": 0.55224609375, "learning_rate": 8.174596119691895e-07, "loss": 0.0056, "reward": 0.2916666716337204, "reward_std": 0.4701542407274246, "rewards/accuracy_reward": 0.2916666716337204, "rewards/format_reward": 0.0, "step": 1028 }, { "clip_ratio": 0.0, "completion_length": 376.4583435058594, "epoch": 0.686, "grad_norm": 0.854775607585907, "kl": 0.194580078125, "learning_rate": 8.143531094119592e-07, "loss": 0.1066, "reward": 0.5000000149011612, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.5000000149011612, "rewards/format_reward": 0.0, "step": 1029 }, { "clip_ratio": 0.0, "completion_length": 620.8333435058594, "epoch": 0.6866666666666666, "grad_norm": 0.42041558027267456, "kl": 1.326171875, "learning_rate": 8.112503199194822e-07, "loss": 0.2049, "reward": 0.2083333358168602, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.2083333358168602, "rewards/format_reward": 0.0, "step": 1030 }, { "clip_ratio": 0.0, "completion_length": 540.5, "epoch": 0.6873333333333334, "grad_norm": 0.30007505416870117, "kl": 0.588623046875, "learning_rate": 8.081512602946639e-07, "loss": 0.0918, "reward": 0.3333333358168602, "reward_std": 0.20412415266036987, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 1031 }, { "clip_ratio": 0.0, "completion_length": 532.3333587646484, "epoch": 0.688, "grad_norm": 0.7963637113571167, "kl": 0.8641357421875, "learning_rate": 8.050559473202078e-07, "loss": 0.1788, "reward": 0.5416666679084301, "reward_std": 0.37592335790395737, "rewards/accuracy_reward": 0.5416666679084301, "rewards/format_reward": 0.0, "step": 1032 }, { "clip_ratio": 0.0, "completion_length": 662.1250076293945, "epoch": 0.6886666666666666, "grad_norm": 0.8141237497329712, "kl": 1.61798095703125, "learning_rate": 8.019643977585277e-07, "loss": 0.2072, "reward": 0.125, "reward_std": 0.1369306445121765, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "step": 1033 }, { "clip_ratio": 0.0, "completion_length": 532.2500076293945, "epoch": 0.6893333333333334, "grad_norm": 0.6974373459815979, "kl": 1.1031494140625, "learning_rate": 7.988766283516587e-07, "loss": 0.1464, "reward": 0.4166666679084301, "reward_std": 0.23899272084236145, "rewards/accuracy_reward": 0.4166666679084301, "rewards/format_reward": 0.0, "step": 1034 }, { "clip_ratio": 0.0, "completion_length": 673.5833435058594, "epoch": 0.69, "grad_norm": 1.9123278856277466, "kl": 1.716796875, "learning_rate": 7.957926558211642e-07, "loss": 0.0789, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 1035 }, { "clip_ratio": 0.0, "completion_length": 553.1666870117188, "epoch": 0.6906666666666667, "grad_norm": 0.6522666811943054, "kl": 0.967529296875, "learning_rate": 7.927124968680447e-07, "loss": 0.2176, "reward": 0.4583333432674408, "reward_std": 0.39512956142425537, "rewards/accuracy_reward": 0.4583333432674408, "rewards/format_reward": 0.0, "step": 1036 }, { "clip_ratio": 0.0, "completion_length": 615.5416946411133, "epoch": 0.6913333333333334, "grad_norm": 0.8828166127204895, "kl": 1.3447265625, "learning_rate": 7.8963616817265e-07, "loss": 0.1022, "reward": 0.0416666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 1037 }, { "clip_ratio": 0.0, "completion_length": 670.5000305175781, "epoch": 0.692, "grad_norm": 2.573199987411499, "kl": 2.818359375, "learning_rate": 7.86563686394587e-07, "loss": 0.1196, "reward": 0.125, "reward_std": 0.1369306445121765, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "step": 1038 }, { "clip_ratio": 0.0, "completion_length": 708.3333435058594, "epoch": 0.6926666666666667, "grad_norm": 0.9883905649185181, "kl": 2.916015625, "learning_rate": 7.834950681726305e-07, "loss": 0.1458, "reward": 0.2916666716337204, "reward_std": 0.39512956142425537, "rewards/accuracy_reward": 0.2916666716337204, "rewards/format_reward": 0.0, "step": 1039 }, { "clip_ratio": 0.0, "completion_length": 563.7500152587891, "epoch": 0.6933333333333334, "grad_norm": 0.6598525643348694, "kl": 1.69921875, "learning_rate": 7.804303301246311e-07, "loss": 0.1789, "reward": 0.3333333432674408, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 1040 }, { "clip_ratio": 0.0, "completion_length": 801.6666870117188, "epoch": 0.694, "grad_norm": 0.5481520295143127, "kl": 1.087158203125, "learning_rate": 7.773694888474268e-07, "loss": 0.0754, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 1041 }, { "clip_ratio": 0.0, "completion_length": 480.9166717529297, "epoch": 0.6946666666666667, "grad_norm": 0.4238288402557373, "kl": 1.1796875, "learning_rate": 7.743125609167557e-07, "loss": 0.1618, "reward": 0.3333333469927311, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.3333333469927311, "rewards/format_reward": 0.0, "step": 1042 }, { "clip_ratio": 0.0, "completion_length": 588.1666870117188, "epoch": 0.6953333333333334, "grad_norm": 0.36901018023490906, "kl": 0.662841796875, "learning_rate": 7.712595628871602e-07, "loss": 0.0924, "reward": 0.3333333432674408, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 1043 }, { "clip_ratio": 0.0, "completion_length": 413.5833435058594, "epoch": 0.696, "grad_norm": 0.31698039174079895, "kl": 0.81475830078125, "learning_rate": 7.682105112919007e-07, "loss": -0.0347, "reward": 0.2916666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.2916666679084301, "rewards/format_reward": 0.0, "step": 1044 }, { "clip_ratio": 0.0, "completion_length": 793.2500152587891, "epoch": 0.6966666666666667, "grad_norm": 0.7039076089859009, "kl": 1.302734375, "learning_rate": 7.651654226428697e-07, "loss": 0.0638, "reward": 0.1666666679084301, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 1045 }, { "clip_ratio": 0.0, "completion_length": 826.9583435058594, "epoch": 0.6973333333333334, "grad_norm": 0.8359678983688354, "kl": 1.054931640625, "learning_rate": 7.621243134304941e-07, "loss": 0.0432, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 1046 }, { "clip_ratio": 0.0, "completion_length": 470.7916717529297, "epoch": 0.698, "grad_norm": 1.593066930770874, "kl": 0.70068359375, "learning_rate": 7.590872001236538e-07, "loss": 0.2298, "reward": 0.4166666828095913, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.4166666828095913, "rewards/format_reward": 0.0, "step": 1047 }, { "clip_ratio": 0.0, "completion_length": 704.7916946411133, "epoch": 0.6986666666666667, "grad_norm": 0.46919992566108704, "kl": 0.5584716796875, "learning_rate": 7.56054099169587e-07, "loss": -0.0275, "reward": 0.2500000037252903, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.0, "step": 1048 }, { "clip_ratio": 0.0, "completion_length": 562.2500076293945, "epoch": 0.6993333333333334, "grad_norm": 1.3073887825012207, "kl": 0.8330078125, "learning_rate": 7.530250269938047e-07, "loss": 0.0989, "reward": 0.3750000111758709, "reward_std": 0.3602609932422638, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 1049 }, { "clip_ratio": 0.0, "completion_length": 431.50002670288086, "epoch": 0.7, "grad_norm": 0.8704689145088196, "kl": 0.314208984375, "learning_rate": 7.500000000000003e-07, "loss": 0.1183, "reward": 0.541666679084301, "reward_std": 0.5242290198802948, "rewards/accuracy_reward": 0.541666679084301, "rewards/format_reward": 0.0, "step": 1050 }, { "clip_ratio": 0.0, "completion_length": 564.5833587646484, "epoch": 0.7006666666666667, "grad_norm": 0.4360133111476898, "kl": 0.45654296875, "learning_rate": 7.469790345699597e-07, "loss": 0.102, "reward": 0.2500000111758709, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.2500000111758709, "rewards/format_reward": 0.0, "step": 1051 }, { "clip_ratio": 0.0, "completion_length": 544.0416717529297, "epoch": 0.7013333333333334, "grad_norm": 1.338711142539978, "kl": 0.469970703125, "learning_rate": 7.43962147063475e-07, "loss": 0.1266, "reward": 0.3750000149011612, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.3750000149011612, "rewards/format_reward": 0.0, "step": 1052 }, { "clip_ratio": 0.0, "completion_length": 689.4583435058594, "epoch": 0.702, "grad_norm": 0.48104557394981384, "kl": 0.61920166015625, "learning_rate": 7.409493538182545e-07, "loss": 0.0448, "reward": 0.2500000074505806, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 1053 }, { "clip_ratio": 0.0, "completion_length": 519.7083511352539, "epoch": 0.7026666666666667, "grad_norm": 1.2438435554504395, "kl": 0.75, "learning_rate": 7.379406711498334e-07, "loss": 0.0739, "reward": 0.2916666679084301, "reward_std": 0.37592335790395737, "rewards/accuracy_reward": 0.2916666679084301, "rewards/format_reward": 0.0, "step": 1054 }, { "clip_ratio": 0.0, "completion_length": 532.7500152587891, "epoch": 0.7033333333333334, "grad_norm": 0.40834712982177734, "kl": 0.73583984375, "learning_rate": 7.349361153514871e-07, "loss": 0.0638, "reward": 0.1666666716337204, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 1055 }, { "clip_ratio": 0.0, "completion_length": 479.37501525878906, "epoch": 0.704, "grad_norm": 0.3739137053489685, "kl": 0.261474609375, "learning_rate": 7.319357026941429e-07, "loss": -0.0288, "reward": 0.2500000111758709, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.2500000111758709, "rewards/format_reward": 0.0, "step": 1056 }, { "clip_ratio": 0.0, "completion_length": 518.3750152587891, "epoch": 0.7046666666666667, "grad_norm": 0.42796579003334045, "kl": 0.516357421875, "learning_rate": 7.289394494262896e-07, "loss": 0.0539, "reward": 0.4583333395421505, "reward_std": 0.3602609932422638, "rewards/accuracy_reward": 0.4583333395421505, "rewards/format_reward": 0.0, "step": 1057 }, { "clip_ratio": 0.0, "completion_length": 612.8333587646484, "epoch": 0.7053333333333334, "grad_norm": 0.8092373609542847, "kl": 0.8193359375, "learning_rate": 7.259473717738927e-07, "loss": 0.1229, "reward": 0.291666679084301, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.291666679084301, "rewards/format_reward": 0.0, "step": 1058 }, { "clip_ratio": 0.0, "completion_length": 729.7500152587891, "epoch": 0.706, "grad_norm": 1.356143832206726, "kl": 1.037109375, "learning_rate": 7.229594859403049e-07, "loss": 0.0573, "reward": 0.1250000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 1059 }, { "clip_ratio": 0.0, "completion_length": 448.75000762939453, "epoch": 0.7066666666666667, "grad_norm": 0.47933605313301086, "kl": 1.0234375, "learning_rate": 7.19975808106177e-07, "loss": -0.1296, "reward": 0.541666679084301, "reward_std": 0.3061862140893936, "rewards/accuracy_reward": 0.541666679084301, "rewards/format_reward": 0.0, "step": 1060 }, { "clip_ratio": 0.0, "completion_length": 430.1666717529297, "epoch": 0.7073333333333334, "grad_norm": 2.3340954780578613, "kl": 0.360107421875, "learning_rate": 7.16996354429373e-07, "loss": 0.1708, "reward": 0.6666666828095913, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.6666666828095913, "rewards/format_reward": 0.0, "step": 1061 }, { "clip_ratio": 0.0, "completion_length": 695.7500228881836, "epoch": 0.708, "grad_norm": 0.7508575916290283, "kl": 0.826904296875, "learning_rate": 7.140211410448815e-07, "loss": 0.061, "reward": 0.2916666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.2916666679084301, "rewards/format_reward": 0.0, "step": 1062 }, { "clip_ratio": 0.0, "completion_length": 530.5833511352539, "epoch": 0.7086666666666667, "grad_norm": 0.39886370301246643, "kl": 0.4774169921875, "learning_rate": 7.11050184064727e-07, "loss": 0.1134, "reward": 0.5000000111758709, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.5000000111758709, "rewards/format_reward": 0.0, "step": 1063 }, { "clip_ratio": 0.0, "completion_length": 439.5416793823242, "epoch": 0.7093333333333334, "grad_norm": 0.8955248594284058, "kl": 1.0341796875, "learning_rate": 7.080834995778853e-07, "loss": 0.2381, "reward": 0.541666679084301, "reward_std": 0.5242290198802948, "rewards/accuracy_reward": 0.541666679084301, "rewards/format_reward": 0.0, "step": 1064 }, { "clip_ratio": 0.0, "completion_length": 577.7083435058594, "epoch": 0.71, "grad_norm": 0.6470255255699158, "kl": 0.6195068359375, "learning_rate": 7.051211036501928e-07, "loss": 0.0306, "reward": 0.3750000037252903, "reward_std": 0.3061862140893936, "rewards/accuracy_reward": 0.3750000037252903, "rewards/format_reward": 0.0, "step": 1065 }, { "clip_ratio": 0.0, "completion_length": 615.0416870117188, "epoch": 0.7106666666666667, "grad_norm": 0.4815261960029602, "kl": 1.040283203125, "learning_rate": 7.021630123242636e-07, "loss": 0.098, "reward": 0.2916666716337204, "reward_std": 0.4701542407274246, "rewards/accuracy_reward": 0.2916666716337204, "rewards/format_reward": 0.0, "step": 1066 }, { "clip_ratio": 0.0, "completion_length": 544.7083511352539, "epoch": 0.7113333333333334, "grad_norm": 0.29296356439590454, "kl": 0.55853271484375, "learning_rate": 6.992092416194e-07, "loss": 0.1449, "reward": 0.2500000074505806, "reward_std": 0.46232306212186813, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 1067 }, { "clip_ratio": 0.0, "completion_length": 613.0000152587891, "epoch": 0.712, "grad_norm": 0.5648375749588013, "kl": 0.7197265625, "learning_rate": 6.962598075315047e-07, "loss": 0.1035, "reward": 0.3333333395421505, "reward_std": 0.3680921792984009, "rewards/accuracy_reward": 0.3333333395421505, "rewards/format_reward": 0.0, "step": 1068 }, { "clip_ratio": 0.0, "completion_length": 672.4166717529297, "epoch": 0.7126666666666667, "grad_norm": 0.696010947227478, "kl": 1.2265625, "learning_rate": 6.933147260329993e-07, "loss": 0.122, "reward": 0.25, "reward_std": 0.273861289024353, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "step": 1069 }, { "clip_ratio": 0.0, "completion_length": 506.41668701171875, "epoch": 0.7133333333333334, "grad_norm": 0.9184134602546692, "kl": 0.88916015625, "learning_rate": 6.903740130727312e-07, "loss": 0.2542, "reward": 0.3333333432674408, "reward_std": 0.3872983753681183, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 1070 }, { "clip_ratio": 0.0, "completion_length": 575.4583549499512, "epoch": 0.714, "grad_norm": 0.2712061107158661, "kl": 0.6602783203125, "learning_rate": 6.874376845758904e-07, "loss": 0.0595, "reward": 0.3750000037252903, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.3750000037252903, "rewards/format_reward": 0.0, "step": 1071 }, { "clip_ratio": 0.0, "completion_length": 639.7500076293945, "epoch": 0.7146666666666667, "grad_norm": 0.8689962029457092, "kl": 0.61962890625, "learning_rate": 6.845057564439251e-07, "loss": 0.1119, "reward": 0.4166666753590107, "reward_std": 0.4971916303038597, "rewards/accuracy_reward": 0.4166666753590107, "rewards/format_reward": 0.0, "step": 1072 }, { "clip_ratio": 0.0, "completion_length": 595.5833435058594, "epoch": 0.7153333333333334, "grad_norm": 1.7736055850982666, "kl": 1.332275390625, "learning_rate": 6.81578244554453e-07, "loss": 0.1822, "reward": 0.2916666716337204, "reward_std": 0.39512956142425537, "rewards/accuracy_reward": 0.2916666716337204, "rewards/format_reward": 0.0, "step": 1073 }, { "clip_ratio": 0.0, "completion_length": 589.9583587646484, "epoch": 0.716, "grad_norm": 0.331196129322052, "kl": 0.5322265625, "learning_rate": 6.786551647611748e-07, "loss": 0.1191, "reward": 0.2500000074505806, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 1074 }, { "clip_ratio": 0.0, "completion_length": 430.29168701171875, "epoch": 0.7166666666666667, "grad_norm": 0.9680533409118652, "kl": 0.91796875, "learning_rate": 6.75736532893791e-07, "loss": 0.1273, "reward": 0.541666679084301, "reward_std": 0.5050228238105774, "rewards/accuracy_reward": 0.541666679084301, "rewards/format_reward": 0.0, "step": 1075 }, { "clip_ratio": 0.0, "completion_length": 623.0833435058594, "epoch": 0.7173333333333334, "grad_norm": 0.39775925874710083, "kl": 1.1171875, "learning_rate": 6.72822364757914e-07, "loss": 0.1026, "reward": 0.2083333358168602, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.2083333358168602, "rewards/format_reward": 0.0, "step": 1076 }, { "clip_ratio": 0.0, "completion_length": 709.2916870117188, "epoch": 0.718, "grad_norm": 0.3493388891220093, "kl": 0.50146484375, "learning_rate": 6.69912676134984e-07, "loss": 0.1068, "reward": 0.1250000037252903, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 1077 }, { "clip_ratio": 0.0, "completion_length": 522.2916717529297, "epoch": 0.7186666666666667, "grad_norm": 0.19044478237628937, "kl": 0.348388671875, "learning_rate": 6.670074827821818e-07, "loss": -0.0079, "reward": 0.5416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.5416666679084301, "rewards/format_reward": 0.0, "step": 1078 }, { "clip_ratio": 0.0, "completion_length": 397.50000762939453, "epoch": 0.7193333333333334, "grad_norm": 1.8943064212799072, "kl": 2.067626953125, "learning_rate": 6.641068004323435e-07, "loss": -0.0026, "reward": 0.7083333507180214, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.7083333507180214, "rewards/format_reward": 0.0, "step": 1079 }, { "clip_ratio": 0.0, "completion_length": 618.5833587646484, "epoch": 0.72, "grad_norm": 0.9424951672554016, "kl": 0.91015625, "learning_rate": 6.6121064479388e-07, "loss": 0.158, "reward": 0.4583333507180214, "reward_std": 0.3602609932422638, "rewards/accuracy_reward": 0.4583333507180214, "rewards/format_reward": 0.0, "step": 1080 }, { "clip_ratio": 0.0, "completion_length": 738.7083587646484, "epoch": 0.7206666666666667, "grad_norm": 0.9115613698959351, "kl": 1.7568359375, "learning_rate": 6.583190315506837e-07, "loss": 0.1663, "reward": 0.1250000037252903, "reward_std": 0.3061862215399742, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 1081 }, { "clip_ratio": 0.0, "completion_length": 487.7083549499512, "epoch": 0.7213333333333334, "grad_norm": 0.8609063625335693, "kl": 1.352294921875, "learning_rate": 6.554319763620506e-07, "loss": 0.1576, "reward": 0.7083333432674408, "reward_std": 0.39512956142425537, "rewards/accuracy_reward": 0.7083333432674408, "rewards/format_reward": 0.0, "step": 1082 }, { "clip_ratio": 0.0, "completion_length": 501.16668701171875, "epoch": 0.722, "grad_norm": 0.39805227518081665, "kl": 0.71234130859375, "learning_rate": 6.525494948625932e-07, "loss": 0.1046, "reward": 0.3333333358168602, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 1083 }, { "clip_ratio": 0.0, "completion_length": 561.708366394043, "epoch": 0.7226666666666667, "grad_norm": 1.5327420234680176, "kl": 2.033203125, "learning_rate": 6.496716026621535e-07, "loss": 0.151, "reward": 0.25, "reward_std": 0.273861289024353, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "step": 1084 }, { "clip_ratio": 0.0, "completion_length": 500.0833511352539, "epoch": 0.7233333333333334, "grad_norm": 1.0278503894805908, "kl": 0.5731201171875, "learning_rate": 6.467983153457227e-07, "loss": 0.0475, "reward": 0.2083333395421505, "reward_std": 0.3602609932422638, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 1085 }, { "clip_ratio": 0.0, "completion_length": 471.6250228881836, "epoch": 0.724, "grad_norm": 0.9015741348266602, "kl": 1.1754150390625, "learning_rate": 6.439296484733526e-07, "loss": 0.256, "reward": 0.6250000111758709, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.6250000111758709, "rewards/format_reward": 0.0, "step": 1086 }, { "clip_ratio": 0.0, "completion_length": 592.5000076293945, "epoch": 0.7246666666666667, "grad_norm": 0.2971435785293579, "kl": 0.531494140625, "learning_rate": 6.41065617580075e-07, "loss": 0.0285, "reward": 0.3333333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 1087 }, { "clip_ratio": 0.0, "completion_length": 464.1666717529297, "epoch": 0.7253333333333334, "grad_norm": 0.7303177118301392, "kl": 0.4913330078125, "learning_rate": 6.38206238175816e-07, "loss": 0.0765, "reward": 0.5833333432674408, "reward_std": 0.3872983753681183, "rewards/accuracy_reward": 0.5833333432674408, "rewards/format_reward": 0.0, "step": 1088 }, { "clip_ratio": 0.0, "completion_length": 565.1666946411133, "epoch": 0.726, "grad_norm": 0.8860296607017517, "kl": 0.55419921875, "learning_rate": 6.353515257453102e-07, "loss": 0.0395, "reward": 0.2500000037252903, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.0, "step": 1089 }, { "clip_ratio": 0.0, "completion_length": 366.0833435058594, "epoch": 0.7266666666666667, "grad_norm": 0.3809724748134613, "kl": 0.46014404296875, "learning_rate": 6.325014957480202e-07, "loss": 0.1352, "reward": 0.5416666716337204, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.5416666716337204, "rewards/format_reward": 0.0, "step": 1090 }, { "clip_ratio": 0.0, "completion_length": 708.4583435058594, "epoch": 0.7273333333333334, "grad_norm": 1.1873146295547485, "kl": 1.68896484375, "learning_rate": 6.296561636180512e-07, "loss": 0.201, "reward": 0.2083333395421505, "reward_std": 0.3602609932422638, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 1091 }, { "clip_ratio": 0.0, "completion_length": 557.1666946411133, "epoch": 0.728, "grad_norm": 0.6245246529579163, "kl": 0.6920166015625, "learning_rate": 6.268155447640661e-07, "loss": -0.0511, "reward": 0.3333333469927311, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.3333333469927311, "rewards/format_reward": 0.0, "step": 1092 }, { "clip_ratio": 0.0, "completion_length": 473.5833435058594, "epoch": 0.7286666666666667, "grad_norm": 1.6437939405441284, "kl": 0.7508544921875, "learning_rate": 6.239796545692046e-07, "loss": 0.1842, "reward": 0.3333333432674408, "reward_std": 0.3872983753681183, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 1093 }, { "clip_ratio": 0.0, "completion_length": 651.7916870117188, "epoch": 0.7293333333333333, "grad_norm": 1.1134110689163208, "kl": 0.91943359375, "learning_rate": 6.211485083909996e-07, "loss": 0.0474, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 1094 }, { "clip_ratio": 0.0, "completion_length": 548.8750114440918, "epoch": 0.73, "grad_norm": 2.540771961212158, "kl": 2.019775390625, "learning_rate": 6.183221215612905e-07, "loss": 0.0981, "reward": 0.291666679084301, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.291666679084301, "rewards/format_reward": 0.0, "step": 1095 }, { "clip_ratio": 0.0, "completion_length": 753.6250076293945, "epoch": 0.7306666666666667, "grad_norm": 0.7461193799972534, "kl": 0.94677734375, "learning_rate": 6.155005093861451e-07, "loss": 0.0459, "reward": 0.0833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 1096 }, { "clip_ratio": 0.0, "completion_length": 546.9583435058594, "epoch": 0.7313333333333333, "grad_norm": 0.2704146206378937, "kl": 0.5679931640625, "learning_rate": 6.126836871457744e-07, "loss": 0.0202, "reward": 0.4583333395421505, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.4583333395421505, "rewards/format_reward": 0.0, "step": 1097 }, { "clip_ratio": 0.0, "completion_length": 467.37500762939453, "epoch": 0.732, "grad_norm": 0.4239635467529297, "kl": 0.70941162109375, "learning_rate": 6.09871670094448e-07, "loss": 0.1366, "reward": 0.4166666716337204, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.4166666716337204, "rewards/format_reward": 0.0, "step": 1098 }, { "clip_ratio": 0.0, "completion_length": 593.3333435058594, "epoch": 0.7326666666666667, "grad_norm": 1.0448439121246338, "kl": 0.555419921875, "learning_rate": 6.070644734604153e-07, "loss": 0.1112, "reward": 0.3750000149011612, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.3750000149011612, "rewards/format_reward": 0.0, "step": 1099 }, { "clip_ratio": 0.0, "completion_length": 478.75001525878906, "epoch": 0.7333333333333333, "grad_norm": 0.27708640694618225, "kl": 0.35284423828125, "learning_rate": 6.04262112445821e-07, "loss": -0.0074, "reward": 0.2500000111758709, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.2500000111758709, "rewards/format_reward": 0.0, "step": 1100 }, { "clip_ratio": 0.0, "completion_length": 391.0416717529297, "epoch": 0.734, "grad_norm": 0.9350998997688293, "kl": 0.243499755859375, "learning_rate": 6.014646022266214e-07, "loss": 0.1002, "reward": 0.7083333544433117, "reward_std": 0.3061862140893936, "rewards/accuracy_reward": 0.7083333544433117, "rewards/format_reward": 0.0, "step": 1101 }, { "clip_ratio": 0.0, "completion_length": 521.4583358764648, "epoch": 0.7346666666666667, "grad_norm": 0.9174404144287109, "kl": 0.927734375, "learning_rate": 5.986719579525055e-07, "loss": 0.1297, "reward": 0.3333333432674408, "reward_std": 0.46232306212186813, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 1102 }, { "clip_ratio": 0.0, "completion_length": 542.7083511352539, "epoch": 0.7353333333333333, "grad_norm": 0.44596433639526367, "kl": 0.79132080078125, "learning_rate": 5.958841947468094e-07, "loss": 0.0652, "reward": 0.3333333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 1103 }, { "clip_ratio": 0.0, "completion_length": 766.9583435058594, "epoch": 0.736, "grad_norm": 1.4180718660354614, "kl": 0.85552978515625, "learning_rate": 5.931013277064378e-07, "loss": 0.0808, "reward": 0.3333333469927311, "reward_std": 0.3332236111164093, "rewards/accuracy_reward": 0.3333333469927311, "rewards/format_reward": 0.0, "step": 1104 }, { "clip_ratio": 0.0, "completion_length": 586.6250228881836, "epoch": 0.7366666666666667, "grad_norm": 0.38047754764556885, "kl": 0.624755859375, "learning_rate": 5.903233719017803e-07, "loss": 0.0647, "reward": 0.4166666679084301, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.4166666679084301, "rewards/format_reward": 0.0, "step": 1105 }, { "clip_ratio": 0.0, "completion_length": 572.8333358764648, "epoch": 0.7373333333333333, "grad_norm": 1.1411820650100708, "kl": 1.0556640625, "learning_rate": 5.875503423766282e-07, "loss": 0.2072, "reward": 0.375, "reward_std": 0.41079193353652954, "rewards/accuracy_reward": 0.375, "rewards/format_reward": 0.0, "step": 1106 }, { "clip_ratio": 0.0, "completion_length": 515.3750152587891, "epoch": 0.738, "grad_norm": 0.17402227222919464, "kl": 0.243896484375, "learning_rate": 5.847822541480986e-07, "loss": 0.0251, "reward": 0.5000000111758709, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.5000000111758709, "rewards/format_reward": 0.0, "step": 1107 }, { "clip_ratio": 0.0, "completion_length": 610.9583435058594, "epoch": 0.7386666666666667, "grad_norm": 0.36285263299942017, "kl": 1.14849853515625, "learning_rate": 5.820191222065462e-07, "loss": 0.0296, "reward": 0.291666679084301, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.291666679084301, "rewards/format_reward": 0.0, "step": 1108 }, { "clip_ratio": 0.0, "completion_length": 622.2500305175781, "epoch": 0.7393333333333333, "grad_norm": 0.7901291847229004, "kl": 0.3590087890625, "learning_rate": 5.792609615154861e-07, "loss": 0.0681, "reward": 0.2083333358168602, "reward_std": 0.3410547897219658, "rewards/accuracy_reward": 0.2083333358168602, "rewards/format_reward": 0.0, "step": 1109 }, { "clip_ratio": 0.0, "completion_length": 467.3750114440918, "epoch": 0.74, "grad_norm": 0.647068440914154, "kl": 0.49755859375, "learning_rate": 5.765077870115125e-07, "loss": 0.0545, "reward": 0.5000000074505806, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.5000000074505806, "rewards/format_reward": 0.0, "step": 1110 }, { "clip_ratio": 0.0, "completion_length": 611.2500076293945, "epoch": 0.7406666666666667, "grad_norm": 1.1274694204330444, "kl": 0.3251953125, "learning_rate": 5.737596136042177e-07, "loss": 0.1656, "reward": 0.5833333544433117, "reward_std": 0.44311685860157013, "rewards/accuracy_reward": 0.5833333544433117, "rewards/format_reward": 0.0, "step": 1111 }, { "clip_ratio": 0.0, "completion_length": 635.2500076293945, "epoch": 0.7413333333333333, "grad_norm": 0.3820166289806366, "kl": 0.9453125, "learning_rate": 5.710164561761102e-07, "loss": 0.0982, "reward": 0.1666666716337204, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 1112 }, { "clip_ratio": 0.0, "completion_length": 646.2083511352539, "epoch": 0.742, "grad_norm": 0.5866655707359314, "kl": 0.71026611328125, "learning_rate": 5.682783295825346e-07, "loss": -0.0106, "reward": 0.2500000037252903, "reward_std": 0.3680921792984009, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.0, "step": 1113 }, { "clip_ratio": 0.0, "completion_length": 564.9166717529297, "epoch": 0.7426666666666667, "grad_norm": 0.8193271160125732, "kl": 0.94970703125, "learning_rate": 5.65545248651592e-07, "loss": 0.1454, "reward": 0.541666679084301, "reward_std": 0.5050228238105774, "rewards/accuracy_reward": 0.541666679084301, "rewards/format_reward": 0.0, "step": 1114 }, { "clip_ratio": 0.0, "completion_length": 650.8750076293945, "epoch": 0.7433333333333333, "grad_norm": 0.8856871128082275, "kl": 0.22509765625, "learning_rate": 5.6281722818406e-07, "loss": 0.0802, "reward": 0.5416666679084301, "reward_std": 0.37592335790395737, "rewards/accuracy_reward": 0.5416666679084301, "rewards/format_reward": 0.0, "step": 1115 }, { "clip_ratio": 0.0, "completion_length": 654.2500076293945, "epoch": 0.744, "grad_norm": 1.0868467092514038, "kl": 1.0374755859375, "learning_rate": 5.600942829533097e-07, "loss": 0.0707, "reward": 0.2500000037252903, "reward_std": 0.3680921792984009, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.0, "step": 1116 }, { "clip_ratio": 0.0, "completion_length": 475.54168701171875, "epoch": 0.7446666666666667, "grad_norm": 0.8178003430366516, "kl": 0.8193359375, "learning_rate": 5.573764277052281e-07, "loss": 0.1952, "reward": 0.5416666753590107, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.5416666753590107, "rewards/format_reward": 0.0, "step": 1117 }, { "clip_ratio": 0.0, "completion_length": 636.4166870117188, "epoch": 0.7453333333333333, "grad_norm": 0.33370211720466614, "kl": 0.5755615234375, "learning_rate": 5.546636771581402e-07, "loss": 0.1781, "reward": 0.2916666716337204, "reward_std": 0.39512956142425537, "rewards/accuracy_reward": 0.2916666716337204, "rewards/format_reward": 0.0, "step": 1118 }, { "clip_ratio": 0.0, "completion_length": 663.3750305175781, "epoch": 0.746, "grad_norm": 0.21963903307914734, "kl": 0.5361328125, "learning_rate": 5.519560460027236e-07, "loss": 0.0604, "reward": 0.1250000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 1119 }, { "clip_ratio": 0.0, "completion_length": 510.08333587646484, "epoch": 0.7466666666666667, "grad_norm": 0.4045485556125641, "kl": 0.743377685546875, "learning_rate": 5.492535489019345e-07, "loss": 0.0852, "reward": 0.4583333358168602, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.4583333358168602, "rewards/format_reward": 0.0, "step": 1120 }, { "clip_ratio": 0.0, "completion_length": 615.0833587646484, "epoch": 0.7473333333333333, "grad_norm": 1.4620929956436157, "kl": 1.9241943359375, "learning_rate": 5.465562004909253e-07, "loss": 0.0981, "reward": 0.2500000111758709, "reward_std": 0.20412415266036987, "rewards/accuracy_reward": 0.2500000111758709, "rewards/format_reward": 0.0, "step": 1121 }, { "clip_ratio": 0.0, "completion_length": 543.3333511352539, "epoch": 0.748, "grad_norm": 0.3096410632133484, "kl": 0.8377685546875, "learning_rate": 5.438640153769653e-07, "loss": 0.0891, "reward": 0.2916666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.2916666679084301, "rewards/format_reward": 0.0, "step": 1122 }, { "clip_ratio": 0.0, "completion_length": 600.7083435058594, "epoch": 0.7486666666666667, "grad_norm": 0.6955377459526062, "kl": 1.73583984375, "learning_rate": 5.41177008139364e-07, "loss": 0.0875, "reward": 0.2083333358168602, "reward_std": 0.3410547822713852, "rewards/accuracy_reward": 0.2083333358168602, "rewards/format_reward": 0.0, "step": 1123 }, { "clip_ratio": 0.0, "completion_length": 552.8750228881836, "epoch": 0.7493333333333333, "grad_norm": 1.032583475112915, "kl": 0.955078125, "learning_rate": 5.384951933293898e-07, "loss": 0.0466, "reward": 0.25, "reward_std": 0.0, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "step": 1124 }, { "clip_ratio": 0.0, "completion_length": 431.6666717529297, "epoch": 0.75, "grad_norm": 0.23286569118499756, "kl": 0.312591552734375, "learning_rate": 5.358185854701909e-07, "loss": 0.0292, "reward": 0.5416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.5416666679084301, "rewards/format_reward": 0.0, "step": 1125 }, { "clip_ratio": 0.0, "completion_length": 514.6250228881836, "epoch": 0.7506666666666667, "grad_norm": 1.5200672149658203, "kl": 0.68743896484375, "learning_rate": 5.331471990567199e-07, "loss": 0.1158, "reward": 0.3750000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.3750000037252903, "rewards/format_reward": 0.0, "step": 1126 }, { "clip_ratio": 0.0, "completion_length": 756.0000152587891, "epoch": 0.7513333333333333, "grad_norm": 0.38066715002059937, "kl": 1.515625, "learning_rate": 5.304810485556506e-07, "loss": 0.1818, "reward": 0.0833333358168602, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 1127 }, { "clip_ratio": 0.0, "completion_length": 446.66668701171875, "epoch": 0.752, "grad_norm": 0.37837380170822144, "kl": 1.52642822265625, "learning_rate": 5.278201484053037e-07, "loss": 0.101, "reward": 0.3333333358168602, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 1128 }, { "clip_ratio": 0.0, "completion_length": 380.7083435058594, "epoch": 0.7526666666666667, "grad_norm": 1.0094460248947144, "kl": 0.12945556640625, "learning_rate": 5.251645130155675e-07, "loss": 0.0726, "reward": 0.6250000149011612, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.6250000149011612, "rewards/format_reward": 0.0, "step": 1129 }, { "clip_ratio": 0.0, "completion_length": 570.3750076293945, "epoch": 0.7533333333333333, "grad_norm": 1.9478181600570679, "kl": 0.852783203125, "learning_rate": 5.225141567678172e-07, "loss": 0.2767, "reward": 0.5416666865348816, "reward_std": 0.4893604442477226, "rewards/accuracy_reward": 0.5416666865348816, "rewards/format_reward": 0.0, "step": 1130 }, { "clip_ratio": 0.0, "completion_length": 600.25, "epoch": 0.754, "grad_norm": 0.9214680790901184, "kl": 1.073974609375, "learning_rate": 5.198690940148418e-07, "loss": 0.068, "reward": 0.2083333432674408, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.2083333432674408, "rewards/format_reward": 0.0, "step": 1131 }, { "clip_ratio": 0.0, "completion_length": 785.1250152587891, "epoch": 0.7546666666666667, "grad_norm": 0.5529047846794128, "kl": 1.10546875, "learning_rate": 5.172293390807627e-07, "loss": 0.0732, "reward": 0.25, "reward_std": 0.273861289024353, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "step": 1132 }, { "clip_ratio": 0.0, "completion_length": 475.33335876464844, "epoch": 0.7553333333333333, "grad_norm": 0.4043007493019104, "kl": 0.53729248046875, "learning_rate": 5.145949062609564e-07, "loss": 0.069, "reward": 0.541666679084301, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.541666679084301, "rewards/format_reward": 0.0, "step": 1133 }, { "clip_ratio": 0.0, "completion_length": 314.54167556762695, "epoch": 0.756, "grad_norm": 0.8321465253829956, "kl": 0.47210693359375, "learning_rate": 5.119658098219792e-07, "loss": -0.0056, "reward": 0.7916666865348816, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.7916666865348816, "rewards/format_reward": 0.0, "step": 1134 }, { "clip_ratio": 0.0, "completion_length": 650.2083511352539, "epoch": 0.7566666666666667, "grad_norm": 1.1938854455947876, "kl": 0.9775390625, "learning_rate": 5.093420640014887e-07, "loss": 0.0817, "reward": 0.3333333395421505, "reward_std": 0.4971916303038597, "rewards/accuracy_reward": 0.3333333395421505, "rewards/format_reward": 0.0, "step": 1135 }, { "clip_ratio": 0.0, "completion_length": 557.5416793823242, "epoch": 0.7573333333333333, "grad_norm": 0.6214411854743958, "kl": 0.7095947265625, "learning_rate": 5.067236830081653e-07, "loss": 0.0439, "reward": 0.3750000149011612, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.3750000149011612, "rewards/format_reward": 0.0, "step": 1136 }, { "clip_ratio": 0.0, "completion_length": 560.7916870117188, "epoch": 0.758, "grad_norm": 0.9473103880882263, "kl": 0.7269287109375, "learning_rate": 5.041106810216376e-07, "loss": 0.1066, "reward": 0.4166666679084301, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.4166666679084301, "rewards/format_reward": 0.0, "step": 1137 }, { "clip_ratio": 0.0, "completion_length": 504.0000228881836, "epoch": 0.7586666666666667, "grad_norm": 0.603123128414154, "kl": 0.7652587890625, "learning_rate": 5.015030721924053e-07, "loss": 0.1269, "reward": 0.3750000149011612, "reward_std": 0.43528567999601364, "rewards/accuracy_reward": 0.3750000149011612, "rewards/format_reward": 0.0, "step": 1138 }, { "clip_ratio": 0.0, "completion_length": 505.25000762939453, "epoch": 0.7593333333333333, "grad_norm": 0.47956621646881104, "kl": 0.623779296875, "learning_rate": 4.989008706417602e-07, "loss": 0.0449, "reward": 0.25, "reward_std": 0.273861289024353, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "step": 1139 }, { "clip_ratio": 0.0, "completion_length": 457.37500762939453, "epoch": 0.76, "grad_norm": 3.3663432598114014, "kl": 0.65234375, "learning_rate": 4.963040904617131e-07, "loss": 0.1133, "reward": 0.5416666716337204, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.5416666716337204, "rewards/format_reward": 0.0, "step": 1140 }, { "clip_ratio": 0.0, "completion_length": 649.7916793823242, "epoch": 0.7606666666666667, "grad_norm": 1.078604817390442, "kl": 1.5751953125, "learning_rate": 4.937127457149138e-07, "loss": 0.0777, "reward": 0.2500000111758709, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.2500000111758709, "rewards/format_reward": 0.0, "step": 1141 }, { "clip_ratio": 0.0, "completion_length": 691.5833587646484, "epoch": 0.7613333333333333, "grad_norm": 0.39219149947166443, "kl": 1.333984375, "learning_rate": 4.911268504345799e-07, "loss": 0.1109, "reward": 0.1250000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 1142 }, { "clip_ratio": 0.0, "completion_length": 508.8333511352539, "epoch": 0.762, "grad_norm": 0.4020535945892334, "kl": 1.0740966796875, "learning_rate": 4.885464186244154e-07, "loss": 0.2332, "reward": 0.4583333469927311, "reward_std": 0.4701542407274246, "rewards/accuracy_reward": 0.4583333469927311, "rewards/format_reward": 0.0, "step": 1143 }, { "clip_ratio": 0.0, "completion_length": 476.00000762939453, "epoch": 0.7626666666666667, "grad_norm": 0.34531015157699585, "kl": 0.98046875, "learning_rate": 4.859714642585368e-07, "loss": 0.1385, "reward": 0.375, "reward_std": 0.1369306445121765, "rewards/accuracy_reward": 0.375, "rewards/format_reward": 0.0, "step": 1144 }, { "clip_ratio": 0.0, "completion_length": 527.2083492279053, "epoch": 0.7633333333333333, "grad_norm": 0.9920143485069275, "kl": 0.9967041015625, "learning_rate": 4.834020012814017e-07, "loss": 0.1005, "reward": 0.5000000037252903, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.5000000037252903, "rewards/format_reward": 0.0, "step": 1145 }, { "clip_ratio": 0.0, "completion_length": 486.25001525878906, "epoch": 0.764, "grad_norm": 0.9626723527908325, "kl": 1.0, "learning_rate": 4.808380436077256e-07, "loss": 0.1053, "reward": 0.3750000074505806, "reward_std": 0.4701542556285858, "rewards/accuracy_reward": 0.3750000074505806, "rewards/format_reward": 0.0, "step": 1146 }, { "clip_ratio": 0.0, "completion_length": 610.3333511352539, "epoch": 0.7646666666666667, "grad_norm": 0.5146697759628296, "kl": 1.421875, "learning_rate": 4.782796051224117e-07, "loss": 0.1613, "reward": 0.1666666716337204, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 1147 }, { "clip_ratio": 0.0, "completion_length": 487.2500190734863, "epoch": 0.7653333333333333, "grad_norm": 0.4963166415691376, "kl": 0.428955078125, "learning_rate": 4.757266996804751e-07, "loss": 0.0758, "reward": 0.4583333395421505, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.4583333395421505, "rewards/format_reward": 0.0, "step": 1148 }, { "clip_ratio": 0.0, "completion_length": 666.2916870117188, "epoch": 0.766, "grad_norm": 1.5027278661727905, "kl": 1.349609375, "learning_rate": 4.731793411069669e-07, "loss": 0.1739, "reward": 0.5000000149011612, "reward_std": 0.3680921792984009, "rewards/accuracy_reward": 0.5000000149011612, "rewards/format_reward": 0.0, "step": 1149 }, { "clip_ratio": 0.0, "completion_length": 446.5416793823242, "epoch": 0.7666666666666667, "grad_norm": 1.1483955383300781, "kl": 1.2904052734375, "learning_rate": 4.7063754319689976e-07, "loss": 0.0926, "reward": 0.4166666828095913, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.4166666828095913, "rewards/format_reward": 0.0, "step": 1150 }, { "clip_ratio": 0.0, "completion_length": 781.9583435058594, "epoch": 0.7673333333333333, "grad_norm": 1.3563785552978516, "kl": 2.0693359375, "learning_rate": 4.681013197151721e-07, "loss": 0.1853, "reward": 0.0833333358168602, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 1151 }, { "clip_ratio": 0.0, "completion_length": 538.8333435058594, "epoch": 0.768, "grad_norm": 0.43684855103492737, "kl": 1.0458984375, "learning_rate": 4.6557068439649533e-07, "loss": 0.0863, "reward": 0.5833333432674408, "reward_std": 0.3872983753681183, "rewards/accuracy_reward": 0.5833333432674408, "rewards/format_reward": 0.0, "step": 1152 }, { "clip_ratio": 0.0, "completion_length": 811.4583435058594, "epoch": 0.7686666666666667, "grad_norm": 0.6081703305244446, "kl": 1.775390625, "learning_rate": 4.6304565094531874e-07, "loss": 0.2647, "reward": 0.1666666679084301, "reward_std": 0.23899272084236145, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 1153 }, { "clip_ratio": 0.0, "completion_length": 456.50000762939453, "epoch": 0.7693333333333333, "grad_norm": 0.8795095682144165, "kl": 0.8717041015625, "learning_rate": 4.6052623303575427e-07, "loss": 0.1616, "reward": 0.5000000074505806, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.5000000074505806, "rewards/format_reward": 0.0, "step": 1154 }, { "clip_ratio": 0.0, "completion_length": 426.08335876464844, "epoch": 0.77, "grad_norm": 2.3582875728607178, "kl": 1.74517822265625, "learning_rate": 4.5801244431150397e-07, "loss": 0.2119, "reward": 0.7083333656191826, "reward_std": 0.43528567254543304, "rewards/accuracy_reward": 0.7083333656191826, "rewards/format_reward": 0.0, "step": 1155 }, { "clip_ratio": 0.0, "completion_length": 505.83334732055664, "epoch": 0.7706666666666667, "grad_norm": 1.2402215003967285, "kl": 1.248779296875, "learning_rate": 4.5550429838578613e-07, "loss": 0.0295, "reward": 0.2500000111758709, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.2500000111758709, "rewards/format_reward": 0.0, "step": 1156 }, { "clip_ratio": 0.0, "completion_length": 696.6666717529297, "epoch": 0.7713333333333333, "grad_norm": 1.6374802589416504, "kl": 2.1787109375, "learning_rate": 4.5300180884125935e-07, "loss": 0.2, "reward": 0.2083333358168602, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.2083333358168602, "rewards/format_reward": 0.0, "step": 1157 }, { "clip_ratio": 0.0, "completion_length": 521.4166870117188, "epoch": 0.772, "grad_norm": 0.8101004362106323, "kl": 0.763427734375, "learning_rate": 4.505049892299517e-07, "loss": 0.1517, "reward": 0.6250000149011612, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.6250000149011612, "rewards/format_reward": 0.0, "step": 1158 }, { "clip_ratio": 0.0, "completion_length": 561.8750133514404, "epoch": 0.7726666666666666, "grad_norm": 0.7721108794212341, "kl": 1.0010986328125, "learning_rate": 4.480138530731867e-07, "loss": 0.1121, "reward": 0.4583333432674408, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.4583333432674408, "rewards/format_reward": 0.0, "step": 1159 }, { "clip_ratio": 0.0, "completion_length": 558.7500152587891, "epoch": 0.7733333333333333, "grad_norm": 1.1660021543502808, "kl": 1.052734375, "learning_rate": 4.4552841386150737e-07, "loss": 0.0435, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 1160 }, { "clip_ratio": 0.0, "completion_length": 618.4166870117188, "epoch": 0.774, "grad_norm": 1.060762643814087, "kl": 1.73828125, "learning_rate": 4.4304868505460793e-07, "loss": 0.1643, "reward": 0.2083333395421505, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 1161 }, { "clip_ratio": 0.0, "completion_length": 528.0000152587891, "epoch": 0.7746666666666666, "grad_norm": 0.4243520498275757, "kl": 1.400390625, "learning_rate": 4.405746800812571e-07, "loss": 0.1099, "reward": 0.1666666679084301, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 1162 }, { "clip_ratio": 0.0, "completion_length": 453.4583511352539, "epoch": 0.7753333333333333, "grad_norm": 2.1658403873443604, "kl": 1.345703125, "learning_rate": 4.381064123392265e-07, "loss": 0.2774, "reward": 0.6250000298023224, "reward_std": 0.3061862140893936, "rewards/accuracy_reward": 0.6250000298023224, "rewards/format_reward": 0.0, "step": 1163 }, { "clip_ratio": 0.0, "completion_length": 655.0417022705078, "epoch": 0.776, "grad_norm": 2.812940835952759, "kl": 2.2265625, "learning_rate": 4.3564389519521896e-07, "loss": 0.1052, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 1164 }, { "clip_ratio": 0.0, "completion_length": 498.9583435058594, "epoch": 0.7766666666666666, "grad_norm": 0.24932549893856049, "kl": 0.1976318359375, "learning_rate": 4.3318714198479437e-07, "loss": 0.032, "reward": 0.4166666716337204, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.4166666716337204, "rewards/format_reward": 0.0, "step": 1165 }, { "clip_ratio": 0.0, "completion_length": 516.5416717529297, "epoch": 0.7773333333333333, "grad_norm": 2.4478025436401367, "kl": 0.9027099609375, "learning_rate": 4.307361660122996e-07, "loss": 0.1085, "reward": 0.4583333358168602, "reward_std": 0.3410547822713852, "rewards/accuracy_reward": 0.4583333358168602, "rewards/format_reward": 0.0, "step": 1166 }, { "clip_ratio": 0.0, "completion_length": 361.08333587646484, "epoch": 0.778, "grad_norm": 0.968588650226593, "kl": 0.81634521484375, "learning_rate": 4.282909805507953e-07, "loss": 0.1197, "reward": 0.5416666679084301, "reward_std": 0.37592335790395737, "rewards/accuracy_reward": 0.5416666679084301, "rewards/format_reward": 0.0, "step": 1167 }, { "clip_ratio": 0.0, "completion_length": 607.7083511352539, "epoch": 0.7786666666666666, "grad_norm": 0.5705548524856567, "kl": 0.6986083984375, "learning_rate": 4.258515988419827e-07, "loss": 0.0526, "reward": 0.3333333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 1168 }, { "clip_ratio": 0.0, "completion_length": 410.00001525878906, "epoch": 0.7793333333333333, "grad_norm": 0.5962058305740356, "kl": 0.8934326171875, "learning_rate": 4.2341803409613494e-07, "loss": 0.1334, "reward": 0.4166666828095913, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.4166666828095913, "rewards/format_reward": 0.0, "step": 1169 }, { "clip_ratio": 0.0, "completion_length": 413.00000762939453, "epoch": 0.78, "grad_norm": 1.6247897148132324, "kl": 0.66796875, "learning_rate": 4.2099029949202353e-07, "loss": 0.1253, "reward": 0.3333333432674408, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 1170 }, { "clip_ratio": 0.0, "completion_length": 503.25, "epoch": 0.7806666666666666, "grad_norm": 0.6739402413368225, "kl": 0.83575439453125, "learning_rate": 4.185684081768462e-07, "loss": 0.1717, "reward": 0.5000000111758709, "reward_std": 0.4971916303038597, "rewards/accuracy_reward": 0.5000000111758709, "rewards/format_reward": 0.0, "step": 1171 }, { "clip_ratio": 0.0, "completion_length": 663.4166946411133, "epoch": 0.7813333333333333, "grad_norm": 0.3293941617012024, "kl": 0.5625, "learning_rate": 4.161523732661579e-07, "loss": 0.0865, "reward": 0.2083333358168602, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.2083333358168602, "rewards/format_reward": 0.0, "step": 1172 }, { "clip_ratio": 0.0, "completion_length": 499.0833435058594, "epoch": 0.782, "grad_norm": 3.859029769897461, "kl": 2.085205078125, "learning_rate": 4.1374220784379914e-07, "loss": 0.1742, "reward": 0.6250000223517418, "reward_std": 0.4701542407274246, "rewards/accuracy_reward": 0.6250000223517418, "rewards/format_reward": 0.0, "step": 1173 }, { "clip_ratio": 0.0, "completion_length": 604.1250152587891, "epoch": 0.7826666666666666, "grad_norm": 0.5492749810218811, "kl": 0.6318359375, "learning_rate": 4.1133792496182266e-07, "loss": 0.0642, "reward": 0.3750000074505806, "reward_std": 0.39512956142425537, "rewards/accuracy_reward": 0.3750000074505806, "rewards/format_reward": 0.0, "step": 1174 }, { "clip_ratio": 0.0, "completion_length": 596.3333435058594, "epoch": 0.7833333333333333, "grad_norm": 0.5458605885505676, "kl": 0.658447265625, "learning_rate": 4.089395376404269e-07, "loss": 0.1314, "reward": 0.2500000074505806, "reward_std": 0.46232306212186813, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 1175 }, { "clip_ratio": 0.0, "completion_length": 519.2500114440918, "epoch": 0.784, "grad_norm": 1.6232075691223145, "kl": 1.0810546875, "learning_rate": 4.06547058867883e-07, "loss": 0.309, "reward": 0.4583333358168602, "reward_std": 0.539891391992569, "rewards/accuracy_reward": 0.4583333358168602, "rewards/format_reward": 0.0, "step": 1176 }, { "clip_ratio": 0.0, "completion_length": 426.7916793823242, "epoch": 0.7846666666666666, "grad_norm": 0.5470168590545654, "kl": 1.42523193359375, "learning_rate": 4.0416050160046336e-07, "loss": 0.226, "reward": 0.416666679084301, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.416666679084301, "rewards/format_reward": 0.0, "step": 1177 }, { "clip_ratio": 0.0, "completion_length": 625.7500076293945, "epoch": 0.7853333333333333, "grad_norm": 0.7003507614135742, "kl": 1.32421875, "learning_rate": 4.017798787623747e-07, "loss": 0.1611, "reward": 0.2500000074505806, "reward_std": 0.46232306212186813, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 1178 }, { "clip_ratio": 0.0, "completion_length": 674.6666870117188, "epoch": 0.786, "grad_norm": 1.227420449256897, "kl": 1.7060546875, "learning_rate": 3.994052032456853e-07, "loss": 0.0862, "reward": 0.1666666716337204, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 1179 }, { "clip_ratio": 0.0, "completion_length": 470.7500114440918, "epoch": 0.7866666666666666, "grad_norm": 0.35155603289604187, "kl": 0.828369140625, "learning_rate": 3.9703648791025716e-07, "loss": 0.0646, "reward": 0.4166666716337204, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.4166666716337204, "rewards/format_reward": 0.0, "step": 1180 }, { "clip_ratio": 0.0, "completion_length": 566.3333587646484, "epoch": 0.7873333333333333, "grad_norm": 0.4553981423377991, "kl": 0.2763671875, "learning_rate": 3.9467374558367437e-07, "loss": 0.0629, "reward": 0.4166666828095913, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.4166666828095913, "rewards/format_reward": 0.0, "step": 1181 }, { "clip_ratio": 0.0, "completion_length": 532.0833473205566, "epoch": 0.788, "grad_norm": 0.6101042032241821, "kl": 1.01513671875, "learning_rate": 3.9231698906117414e-07, "loss": 0.1049, "reward": 0.4583333432674408, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.4583333432674408, "rewards/format_reward": 0.0, "step": 1182 }, { "clip_ratio": 0.0, "completion_length": 570.8333435058594, "epoch": 0.7886666666666666, "grad_norm": 0.5739076733589172, "kl": 0.908203125, "learning_rate": 3.899662311055808e-07, "loss": -0.0222, "reward": 0.3333333469927311, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.3333333469927311, "rewards/format_reward": 0.0, "step": 1183 }, { "clip_ratio": 0.0, "completion_length": 685.208366394043, "epoch": 0.7893333333333333, "grad_norm": 1.000920057296753, "kl": 1.507568359375, "learning_rate": 3.876214844472311e-07, "loss": 0.0544, "reward": 0.1666666679084301, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 1184 }, { "clip_ratio": 0.0, "completion_length": 578.2083511352539, "epoch": 0.79, "grad_norm": 0.6230782866477966, "kl": 1.8408203125, "learning_rate": 3.852827617839085e-07, "loss": 0.1915, "reward": 0.3333333395421505, "reward_std": 0.3680921792984009, "rewards/accuracy_reward": 0.3333333395421505, "rewards/format_reward": 0.0, "step": 1185 }, { "clip_ratio": 0.0, "completion_length": 628.0000152587891, "epoch": 0.7906666666666666, "grad_norm": 3.06282377243042, "kl": 1.18011474609375, "learning_rate": 3.829500757807761e-07, "loss": 0.0737, "reward": 0.1250000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 1186 }, { "clip_ratio": 0.0, "completion_length": 500.4166831970215, "epoch": 0.7913333333333333, "grad_norm": 1.0502980947494507, "kl": 1.27001953125, "learning_rate": 3.806234390703034e-07, "loss": 0.1446, "reward": 0.6250000149011612, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.6250000149011612, "rewards/format_reward": 0.0, "step": 1187 }, { "clip_ratio": 0.0, "completion_length": 507.91668701171875, "epoch": 0.792, "grad_norm": 0.6540380716323853, "kl": 0.4942626953125, "learning_rate": 3.7830286425220237e-07, "loss": -0.0144, "reward": 0.2500000111758709, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.2500000111758709, "rewards/format_reward": 0.0, "step": 1188 }, { "clip_ratio": 0.0, "completion_length": 443.7083549499512, "epoch": 0.7926666666666666, "grad_norm": 1.0590084791183472, "kl": 1.0517578125, "learning_rate": 3.7598836389335566e-07, "loss": 0.0467, "reward": 0.5, "reward_std": 0.0, "rewards/accuracy_reward": 0.5, "rewards/format_reward": 0.0, "step": 1189 }, { "clip_ratio": 0.0, "completion_length": 553.3333396911621, "epoch": 0.7933333333333333, "grad_norm": 1.8041315078735352, "kl": 0.8536376953125, "learning_rate": 3.736799505277512e-07, "loss": 0.0049, "reward": 0.5833333358168602, "reward_std": 0.40296074748039246, "rewards/accuracy_reward": 0.5833333358168602, "rewards/format_reward": 0.0, "step": 1190 }, { "clip_ratio": 0.0, "completion_length": 576.5416793823242, "epoch": 0.794, "grad_norm": 4.757410526275635, "kl": 0.22003173828125, "learning_rate": 3.713776366564136e-07, "loss": -0.0111, "reward": 0.5416666753590107, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.5416666753590107, "rewards/format_reward": 0.0, "step": 1191 }, { "clip_ratio": 0.0, "completion_length": 575.8750152587891, "epoch": 0.7946666666666666, "grad_norm": 0.614162802696228, "kl": 1.0859375, "learning_rate": 3.69081434747335e-07, "loss": 0.1008, "reward": 0.2083333395421505, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 1192 }, { "clip_ratio": 0.0, "completion_length": 619.9166870117188, "epoch": 0.7953333333333333, "grad_norm": 0.6076165437698364, "kl": 0.93536376953125, "learning_rate": 3.667913572354096e-07, "loss": 0.0799, "reward": 0.2916666716337204, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.2916666716337204, "rewards/format_reward": 0.0, "step": 1193 }, { "clip_ratio": 0.0, "completion_length": 569.5833587646484, "epoch": 0.796, "grad_norm": 0.9573508501052856, "kl": 1.02569580078125, "learning_rate": 3.6450741652236556e-07, "loss": 0.1251, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 1194 }, { "clip_ratio": 0.0, "completion_length": 569.6250114440918, "epoch": 0.7966666666666666, "grad_norm": 0.7159854173660278, "kl": 1.10546875, "learning_rate": 3.622296249766967e-07, "loss": 0.1186, "reward": 0.4166666828095913, "reward_std": 0.46232305467128754, "rewards/accuracy_reward": 0.4166666828095913, "rewards/format_reward": 0.0, "step": 1195 }, { "clip_ratio": 0.0, "completion_length": 456.2083549499512, "epoch": 0.7973333333333333, "grad_norm": 0.38320422172546387, "kl": 0.6025390625, "learning_rate": 3.599579949335973e-07, "loss": 0.1333, "reward": 0.3750000037252903, "reward_std": 0.3061862140893936, "rewards/accuracy_reward": 0.3750000037252903, "rewards/format_reward": 0.0, "step": 1196 }, { "clip_ratio": 0.0, "completion_length": 602.7083511352539, "epoch": 0.798, "grad_norm": 0.4707263708114624, "kl": 0.49078369140625, "learning_rate": 3.5769253869489515e-07, "loss": -0.0206, "reward": 0.3750000111758709, "reward_std": 0.3410547822713852, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 1197 }, { "clip_ratio": 0.0, "completion_length": 434.1250114440918, "epoch": 0.7986666666666666, "grad_norm": 0.330828458070755, "kl": 0.319091796875, "learning_rate": 3.5543326852898234e-07, "loss": -0.0213, "reward": 0.5833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.5833333358168602, "rewards/format_reward": 0.0, "step": 1198 }, { "clip_ratio": 0.0, "completion_length": 490.25001525878906, "epoch": 0.7993333333333333, "grad_norm": 0.2754898965358734, "kl": 1.041778564453125, "learning_rate": 3.5318019667075265e-07, "loss": 0.109, "reward": 0.1666666716337204, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 1199 }, { "clip_ratio": 0.0, "completion_length": 373.00000762939453, "epoch": 0.8, "grad_norm": 0.1987455040216446, "kl": 0.102447509765625, "learning_rate": 3.5093333532153313e-07, "loss": 0.003, "reward": 0.5416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.5416666679084301, "rewards/format_reward": 0.0, "step": 1200 }, { "clip_ratio": 0.0, "completion_length": 549.9166679382324, "epoch": 0.8006666666666666, "grad_norm": 2.015596389770508, "kl": 1.0859375, "learning_rate": 3.486926966490171e-07, "loss": 0.0048, "reward": 0.4583333507180214, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.4583333507180214, "rewards/format_reward": 0.0, "step": 1201 }, { "clip_ratio": 0.0, "completion_length": 591.4166793823242, "epoch": 0.8013333333333333, "grad_norm": 0.7710190415382385, "kl": 0.612548828125, "learning_rate": 3.464582927872013e-07, "loss": 0.0814, "reward": 0.25, "reward_std": 0.273861289024353, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "step": 1202 }, { "clip_ratio": 0.0, "completion_length": 552.2083435058594, "epoch": 0.802, "grad_norm": 0.9221892952919006, "kl": 0.4732666015625, "learning_rate": 3.442301358363163e-07, "loss": 0.0701, "reward": 0.5000000074505806, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.5000000074505806, "rewards/format_reward": 0.0, "step": 1203 }, { "clip_ratio": 0.0, "completion_length": 503.3333549499512, "epoch": 0.8026666666666666, "grad_norm": 0.5028128027915955, "kl": 0.53076171875, "learning_rate": 3.4200823786276493e-07, "loss": -0.0032, "reward": 0.3333333358168602, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 1204 }, { "clip_ratio": 0.0, "completion_length": 569.5416870117188, "epoch": 0.8033333333333333, "grad_norm": 1.0794084072113037, "kl": 1.3096923828125, "learning_rate": 3.3979261089905426e-07, "loss": 0.114, "reward": 0.5000000074505806, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.5000000074505806, "rewards/format_reward": 0.0, "step": 1205 }, { "clip_ratio": 0.0, "completion_length": 572.2916870117188, "epoch": 0.804, "grad_norm": 0.8213533163070679, "kl": 1.469818115234375, "learning_rate": 3.3758326694373073e-07, "loss": 0.0076, "reward": 0.3333333469927311, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.3333333469927311, "rewards/format_reward": 0.0, "step": 1206 }, { "clip_ratio": 0.0, "completion_length": 613.8333511352539, "epoch": 0.8046666666666666, "grad_norm": 0.5840124487876892, "kl": 0.935546875, "learning_rate": 3.353802179613164e-07, "loss": 0.0869, "reward": 0.1666666716337204, "reward_std": 0.3332236111164093, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 1207 }, { "clip_ratio": 0.0, "completion_length": 352.3333396911621, "epoch": 0.8053333333333333, "grad_norm": 1.1185375452041626, "kl": 0.4600830078125, "learning_rate": 3.3318347588224366e-07, "loss": 0.0179, "reward": 0.6250000074505806, "reward_std": 0.39512956142425537, "rewards/accuracy_reward": 0.6250000074505806, "rewards/format_reward": 0.0, "step": 1208 }, { "clip_ratio": 0.0, "completion_length": 520.5416793823242, "epoch": 0.806, "grad_norm": 0.9138520359992981, "kl": 0.69879150390625, "learning_rate": 3.309930526027885e-07, "loss": 0.0211, "reward": 0.6250000074505806, "reward_std": 0.39512956142425537, "rewards/accuracy_reward": 0.6250000074505806, "rewards/format_reward": 0.0, "step": 1209 }, { "clip_ratio": 0.0, "completion_length": 519.1250076293945, "epoch": 0.8066666666666666, "grad_norm": 0.7848846912384033, "kl": 1.28515625, "learning_rate": 3.288089599850112e-07, "loss": 0.049, "reward": 0.25, "reward_std": 0.273861289024353, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "step": 1210 }, { "clip_ratio": 0.0, "completion_length": 555.0833435058594, "epoch": 0.8073333333333333, "grad_norm": 0.8577990531921387, "kl": 1.298828125, "learning_rate": 3.266312098566859e-07, "loss": 0.1287, "reward": 0.416666679084301, "reward_std": 0.3872983753681183, "rewards/accuracy_reward": 0.416666679084301, "rewards/format_reward": 0.0, "step": 1211 }, { "clip_ratio": 0.0, "completion_length": 586.8750152587891, "epoch": 0.808, "grad_norm": 1.4690775871276855, "kl": 1.8828125, "learning_rate": 3.2445981401124044e-07, "loss": 0.1163, "reward": 0.2083333358168602, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.2083333358168602, "rewards/format_reward": 0.0, "step": 1212 }, { "clip_ratio": 0.0, "completion_length": 615.7500305175781, "epoch": 0.8086666666666666, "grad_norm": 0.8727515935897827, "kl": 0.959228515625, "learning_rate": 3.222947842076921e-07, "loss": -0.0211, "reward": 0.125, "reward_std": 0.1369306445121765, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "step": 1213 }, { "clip_ratio": 0.0, "completion_length": 510.58335876464844, "epoch": 0.8093333333333333, "grad_norm": 2.31070613861084, "kl": 0.98193359375, "learning_rate": 3.2013613217058323e-07, "loss": 0.1457, "reward": 0.5833333432674408, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.5833333432674408, "rewards/format_reward": 0.0, "step": 1214 }, { "clip_ratio": 0.0, "completion_length": 801.2500305175781, "epoch": 0.81, "grad_norm": 0.9432411789894104, "kl": 1.02056884765625, "learning_rate": 3.1798386958991715e-07, "loss": 0.0709, "reward": 0.3333333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 1215 }, { "clip_ratio": 0.0, "completion_length": 686.4583435058594, "epoch": 0.8106666666666666, "grad_norm": 0.33405911922454834, "kl": 0.8466796875, "learning_rate": 3.1583800812109646e-07, "loss": 0.0905, "reward": 0.2083333358168602, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.2083333358168602, "rewards/format_reward": 0.0, "step": 1216 }, { "clip_ratio": 0.0, "completion_length": 587.6666793823242, "epoch": 0.8113333333333334, "grad_norm": 0.7215322852134705, "kl": 0.650634765625, "learning_rate": 3.136985593848586e-07, "loss": 0.0749, "reward": 0.0833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 1217 }, { "clip_ratio": 0.0, "completion_length": 485.41668701171875, "epoch": 0.812, "grad_norm": 1.2232675552368164, "kl": 2.390380859375, "learning_rate": 3.1156553496721414e-07, "loss": 0.1073, "reward": 0.625, "reward_std": 0.1369306445121765, "rewards/accuracy_reward": 0.625, "rewards/format_reward": 0.0, "step": 1218 }, { "clip_ratio": 0.0, "completion_length": 634.1666793823242, "epoch": 0.8126666666666666, "grad_norm": 0.4092505872249603, "kl": 1.2626953125, "learning_rate": 3.094389464193821e-07, "loss": 0.1182, "reward": 0.4583333395421505, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.4583333395421505, "rewards/format_reward": 0.0, "step": 1219 }, { "clip_ratio": 0.0, "completion_length": 639.8333511352539, "epoch": 0.8133333333333334, "grad_norm": 1.0257701873779297, "kl": 1.66650390625, "learning_rate": 3.073188052577282e-07, "loss": 0.1713, "reward": 0.3333333469927311, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.3333333469927311, "rewards/format_reward": 0.0, "step": 1220 }, { "clip_ratio": 0.0, "completion_length": 512.7916717529297, "epoch": 0.814, "grad_norm": 0.3284270763397217, "kl": 0.981201171875, "learning_rate": 3.052051229637056e-07, "loss": 0.0814, "reward": 0.3333333469927311, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.3333333469927311, "rewards/format_reward": 0.0, "step": 1221 }, { "clip_ratio": 0.0, "completion_length": 568.3750152587891, "epoch": 0.8146666666666667, "grad_norm": 0.34729981422424316, "kl": 0.6260986328125, "learning_rate": 3.030979109837871e-07, "loss": 0.1216, "reward": 0.5000000074505806, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.5000000074505806, "rewards/format_reward": 0.0, "step": 1222 }, { "clip_ratio": 0.0, "completion_length": 596.0833549499512, "epoch": 0.8153333333333334, "grad_norm": 0.3546624481678009, "kl": 0.6287841796875, "learning_rate": 3.0099718072940733e-07, "loss": 0.0348, "reward": 0.5416666753590107, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.5416666753590107, "rewards/format_reward": 0.0, "step": 1223 }, { "clip_ratio": 0.0, "completion_length": 568.5833587646484, "epoch": 0.816, "grad_norm": 0.2776282727718353, "kl": 0.7977294921875, "learning_rate": 2.9890294357689994e-07, "loss": 0.0367, "reward": 0.3333333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 1224 }, { "clip_ratio": 0.0, "completion_length": 492.87501525878906, "epoch": 0.8166666666666667, "grad_norm": 0.5661952495574951, "kl": 1.012451171875, "learning_rate": 2.9681521086743423e-07, "loss": 0.0278, "reward": 0.3333333358168602, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 1225 }, { "clip_ratio": 0.0, "completion_length": 602.2083435058594, "epoch": 0.8173333333333334, "grad_norm": 1.1989102363586426, "kl": 1.4765625, "learning_rate": 2.947339939069574e-07, "loss": 0.0626, "reward": 0.0833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 1226 }, { "clip_ratio": 0.0, "completion_length": 604.3333435058594, "epoch": 0.818, "grad_norm": 1.9877218008041382, "kl": 0.603515625, "learning_rate": 2.9265930396612894e-07, "loss": 0.1388, "reward": 0.5416666865348816, "reward_std": 0.3410547897219658, "rewards/accuracy_reward": 0.5416666865348816, "rewards/format_reward": 0.0, "step": 1227 }, { "clip_ratio": 0.0, "completion_length": 460.2083435058594, "epoch": 0.8186666666666667, "grad_norm": 0.728634238243103, "kl": 0.5634765625, "learning_rate": 2.905911522802633e-07, "loss": 0.1491, "reward": 0.541666679084301, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.541666679084301, "rewards/format_reward": 0.0, "step": 1228 }, { "clip_ratio": 0.0, "completion_length": 499.41668701171875, "epoch": 0.8193333333333334, "grad_norm": 0.8001137971878052, "kl": 0.90966796875, "learning_rate": 2.885295500492674e-07, "loss": 0.0585, "reward": 0.0833333358168602, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 1229 }, { "clip_ratio": 0.0, "completion_length": 725.5000152587891, "epoch": 0.82, "grad_norm": 1.574211835861206, "kl": 0.75634765625, "learning_rate": 2.86474508437579e-07, "loss": 0.0701, "reward": 0.2500000074505806, "reward_std": 0.3872983753681183, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 1230 }, { "clip_ratio": 0.0, "completion_length": 495.87501525878906, "epoch": 0.8206666666666667, "grad_norm": 1.2920063734054565, "kl": 1.16357421875, "learning_rate": 2.8442603857410854e-07, "loss": 0.173, "reward": 0.4166666828095913, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.4166666828095913, "rewards/format_reward": 0.0, "step": 1231 }, { "clip_ratio": 0.0, "completion_length": 463.6666717529297, "epoch": 0.8213333333333334, "grad_norm": 2.08899188041687, "kl": 0.44219970703125, "learning_rate": 2.8238415155217796e-07, "loss": 0.0918, "reward": 0.6666666865348816, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.6666666865348816, "rewards/format_reward": 0.0, "step": 1232 }, { "clip_ratio": 0.0, "completion_length": 633.0416870117188, "epoch": 0.822, "grad_norm": 1.209104299545288, "kl": 0.8046875, "learning_rate": 2.8034885842945865e-07, "loss": 0.1324, "reward": 0.416666679084301, "reward_std": 0.4431168660521507, "rewards/accuracy_reward": 0.416666679084301, "rewards/format_reward": 0.0, "step": 1233 }, { "clip_ratio": 0.0, "completion_length": 620.7916870117188, "epoch": 0.8226666666666667, "grad_norm": 0.9696624279022217, "kl": 1.40234375, "learning_rate": 2.783201702279147e-07, "loss": 0.2426, "reward": 0.2916666753590107, "reward_std": 0.3602609932422638, "rewards/accuracy_reward": 0.2916666753590107, "rewards/format_reward": 0.0, "step": 1234 }, { "clip_ratio": 0.0, "completion_length": 600.8750152587891, "epoch": 0.8233333333333334, "grad_norm": 2.164545774459839, "kl": 1.306640625, "learning_rate": 2.762980979337422e-07, "loss": 0.197, "reward": 0.5416666865348816, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.5416666865348816, "rewards/format_reward": 0.0, "step": 1235 }, { "clip_ratio": 0.0, "completion_length": 491.6666679382324, "epoch": 0.824, "grad_norm": 0.2945857644081116, "kl": 0.372802734375, "learning_rate": 2.7428265249730726e-07, "loss": 0.0144, "reward": 0.5, "reward_std": 0.0, "rewards/accuracy_reward": 0.5, "rewards/format_reward": 0.0, "step": 1236 }, { "clip_ratio": 0.0, "completion_length": 705.9166870117188, "epoch": 0.8246666666666667, "grad_norm": 0.5073980093002319, "kl": 1.6783447265625, "learning_rate": 2.7227384483309085e-07, "loss": 0.0273, "reward": 0.2916666679084301, "reward_std": 0.37592336535453796, "rewards/accuracy_reward": 0.2916666679084301, "rewards/format_reward": 0.0, "step": 1237 }, { "clip_ratio": 0.0, "completion_length": 496.75000762939453, "epoch": 0.8253333333333334, "grad_norm": 0.26658689975738525, "kl": 0.40496826171875, "learning_rate": 2.7027168581962706e-07, "loss": 0.0697, "reward": 0.3333333469927311, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.3333333469927311, "rewards/format_reward": 0.0, "step": 1238 }, { "clip_ratio": 0.0, "completion_length": 720.2500228881836, "epoch": 0.826, "grad_norm": 0.9980387091636658, "kl": 1.1680908203125, "learning_rate": 2.6827618629944393e-07, "loss": 0.0856, "reward": 0.1666666716337204, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 1239 }, { "clip_ratio": 0.0, "completion_length": 406.4166717529297, "epoch": 0.8266666666666667, "grad_norm": 0.3413832485675812, "kl": 0.57794189453125, "learning_rate": 2.6628735707900655e-07, "loss": 0.0474, "reward": 0.2083333358168602, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.2083333358168602, "rewards/format_reward": 0.0, "step": 1240 }, { "clip_ratio": 0.0, "completion_length": 558.7083511352539, "epoch": 0.8273333333333334, "grad_norm": 1.5075721740722656, "kl": 0.518798828125, "learning_rate": 2.6430520892865743e-07, "loss": 0.0144, "reward": 0.3750000111758709, "reward_std": 0.3602609932422638, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 1241 }, { "clip_ratio": 0.0, "completion_length": 661.0000228881836, "epoch": 0.828, "grad_norm": 1.303357720375061, "kl": 1.216796875, "learning_rate": 2.6232975258255744e-07, "loss": 0.1473, "reward": 0.2500000111758709, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.2500000111758709, "rewards/format_reward": 0.0, "step": 1242 }, { "clip_ratio": 0.0, "completion_length": 467.2083435058594, "epoch": 0.8286666666666667, "grad_norm": 1.3202844858169556, "kl": 0.978515625, "learning_rate": 2.603609987386297e-07, "loss": -0.0095, "reward": 0.4583333469927311, "reward_std": 0.4701542481780052, "rewards/accuracy_reward": 0.4583333469927311, "rewards/format_reward": 0.0, "step": 1243 }, { "clip_ratio": 0.0, "completion_length": 379.83333587646484, "epoch": 0.8293333333333334, "grad_norm": 1.1333260536193848, "kl": 0.91278076171875, "learning_rate": 2.5839895805849913e-07, "loss": 0.026, "reward": 0.5000000074505806, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.5000000074505806, "rewards/format_reward": 0.0, "step": 1244 }, { "clip_ratio": 0.0, "completion_length": 696.2916793823242, "epoch": 0.83, "grad_norm": 5.657405376434326, "kl": 1.333740234375, "learning_rate": 2.5644364116743757e-07, "loss": 0.1912, "reward": 0.3333333358168602, "reward_std": 0.40296074748039246, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 1245 }, { "clip_ratio": 0.0, "completion_length": 500.7916793823242, "epoch": 0.8306666666666667, "grad_norm": 1.0479775667190552, "kl": 0.25537109375, "learning_rate": 2.54495058654304e-07, "loss": 0.0959, "reward": 0.2500000111758709, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.2500000111758709, "rewards/format_reward": 0.0, "step": 1246 }, { "clip_ratio": 0.0, "completion_length": 414.79168701171875, "epoch": 0.8313333333333334, "grad_norm": 1.1270136833190918, "kl": 1.05743408203125, "learning_rate": 2.525532210714869e-07, "loss": 0.0424, "reward": 0.4583333395421505, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.4583333395421505, "rewards/format_reward": 0.0, "step": 1247 }, { "clip_ratio": 0.0, "completion_length": 639.0000228881836, "epoch": 0.832, "grad_norm": 0.5474291443824768, "kl": 1.4820556640625, "learning_rate": 2.5061813893485086e-07, "loss": 0.2761, "reward": 0.3750000074505806, "reward_std": 0.39512956142425537, "rewards/accuracy_reward": 0.3750000074505806, "rewards/format_reward": 0.0, "step": 1248 }, { "clip_ratio": 0.0, "completion_length": 530.6250076293945, "epoch": 0.8326666666666667, "grad_norm": 0.9027184247970581, "kl": 1.09765625, "learning_rate": 2.4868982272367406e-07, "loss": 0.0834, "reward": 0.1666666716337204, "reward_std": 0.3332236111164093, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 1249 }, { "clip_ratio": 0.0, "completion_length": 592.7083435058594, "epoch": 0.8333333333333334, "grad_norm": 2.309518337249756, "kl": 3.53125, "learning_rate": 2.467682828805956e-07, "loss": 0.3031, "reward": 0.2500000037252903, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.0, "step": 1250 }, { "clip_ratio": 0.0, "completion_length": 499.2916793823242, "epoch": 0.834, "grad_norm": 1.3370381593704224, "kl": 0.86383056640625, "learning_rate": 2.448535298115576e-07, "loss": 0.2357, "reward": 0.3750000111758709, "reward_std": 0.3602609932422638, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 1251 }, { "clip_ratio": 0.0, "completion_length": 547.2916793823242, "epoch": 0.8346666666666667, "grad_norm": 1.0299237966537476, "kl": 0.631988525390625, "learning_rate": 2.4294557388574894e-07, "loss": 0.1124, "reward": 0.5833333432674408, "reward_std": 0.23899272084236145, "rewards/accuracy_reward": 0.5833333432674408, "rewards/format_reward": 0.0, "step": 1252 }, { "clip_ratio": 0.0, "completion_length": 645.5416870117188, "epoch": 0.8353333333333334, "grad_norm": 1.4836759567260742, "kl": 1.1591796875, "learning_rate": 2.410444254355499e-07, "loss": 0.1404, "reward": 0.3750000111758709, "reward_std": 0.3410547897219658, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 1253 }, { "clip_ratio": 0.0, "completion_length": 281.70834732055664, "epoch": 0.836, "grad_norm": 0.460718035697937, "kl": 0.2930908203125, "learning_rate": 2.3915009475647363e-07, "loss": -0.0432, "reward": 0.5000000111758709, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.5000000111758709, "rewards/format_reward": 0.0, "step": 1254 }, { "clip_ratio": 0.0, "completion_length": 416.8333549499512, "epoch": 0.8366666666666667, "grad_norm": 0.4699859023094177, "kl": 0.379150390625, "learning_rate": 2.372625921071137e-07, "loss": 0.0248, "reward": 0.2500000111758709, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.2500000111758709, "rewards/format_reward": 0.0, "step": 1255 }, { "clip_ratio": 0.0, "completion_length": 631.9583511352539, "epoch": 0.8373333333333334, "grad_norm": 1.2533116340637207, "kl": 2.072845458984375, "learning_rate": 2.3538192770908713e-07, "loss": 0.2083, "reward": 0.2083333358168602, "reward_std": 0.3410547822713852, "rewards/accuracy_reward": 0.2083333358168602, "rewards/format_reward": 0.0, "step": 1256 }, { "clip_ratio": 0.0, "completion_length": 675.5416717529297, "epoch": 0.838, "grad_norm": 0.7279332876205444, "kl": 1.1171875, "learning_rate": 2.3350811174697772e-07, "loss": 0.052, "reward": 0.1666666679084301, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 1257 }, { "clip_ratio": 0.0, "completion_length": 568.7500076293945, "epoch": 0.8386666666666667, "grad_norm": 1.2850632667541504, "kl": 1.67919921875, "learning_rate": 2.3164115436828236e-07, "loss": 0.1108, "reward": 0.1250000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 1258 }, { "clip_ratio": 0.0, "completion_length": 476.0833435058594, "epoch": 0.8393333333333334, "grad_norm": 1.1394065618515015, "kl": 1.48779296875, "learning_rate": 2.2978106568335776e-07, "loss": 0.2331, "reward": 0.5000000149011612, "reward_std": 0.5163978338241577, "rewards/accuracy_reward": 0.5000000149011612, "rewards/format_reward": 0.0, "step": 1259 }, { "clip_ratio": 0.0, "completion_length": 460.79168701171875, "epoch": 0.84, "grad_norm": 0.6185013055801392, "kl": 1.288543701171875, "learning_rate": 2.2792785576536108e-07, "loss": 0.1698, "reward": 0.3750000037252903, "reward_std": 0.306186206638813, "rewards/accuracy_reward": 0.3750000037252903, "rewards/format_reward": 0.0, "step": 1260 }, { "clip_ratio": 0.0, "completion_length": 689.1666870117188, "epoch": 0.8406666666666667, "grad_norm": 1.1795493364334106, "kl": 2.298828125, "learning_rate": 2.260815346501997e-07, "loss": 0.2093, "reward": 0.3333333395421505, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.3333333395421505, "rewards/format_reward": 0.0, "step": 1261 }, { "clip_ratio": 0.0, "completion_length": 634.7500228881836, "epoch": 0.8413333333333334, "grad_norm": 6.540075302124023, "kl": 1.050048828125, "learning_rate": 2.242421123364753e-07, "loss": 0.1673, "reward": 0.3750000111758709, "reward_std": 0.3410547822713852, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 1262 }, { "clip_ratio": 0.0, "completion_length": 537.7500076293945, "epoch": 0.842, "grad_norm": 0.30968618392944336, "kl": 0.60791015625, "learning_rate": 2.224095987854285e-07, "loss": 0.075, "reward": 0.1250000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 1263 }, { "clip_ratio": 0.0, "completion_length": 367.75000762939453, "epoch": 0.8426666666666667, "grad_norm": 0.9896639585494995, "kl": 0.82635498046875, "learning_rate": 2.2058400392088757e-07, "loss": 0.0663, "reward": 0.6666666865348816, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.6666666865348816, "rewards/format_reward": 0.0, "step": 1264 }, { "clip_ratio": 0.0, "completion_length": 729.7916870117188, "epoch": 0.8433333333333334, "grad_norm": 0.7986060976982117, "kl": 1.83203125, "learning_rate": 2.1876533762921175e-07, "loss": 0.1948, "reward": 0.3333333469927311, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.3333333469927311, "rewards/format_reward": 0.0, "step": 1265 }, { "clip_ratio": 0.0, "completion_length": 592.1666870117188, "epoch": 0.844, "grad_norm": 0.9256241917610168, "kl": 1.83209228515625, "learning_rate": 2.169536097592401e-07, "loss": 0.3025, "reward": 0.416666679084301, "reward_std": 0.44311685860157013, "rewards/accuracy_reward": 0.416666679084301, "rewards/format_reward": 0.0, "step": 1266 }, { "clip_ratio": 0.0, "completion_length": 508.62501525878906, "epoch": 0.8446666666666667, "grad_norm": 1.2395111322402954, "kl": 0.728515625, "learning_rate": 2.151488301222373e-07, "loss": 0.0473, "reward": 0.291666679084301, "reward_std": 0.3061862140893936, "rewards/accuracy_reward": 0.291666679084301, "rewards/format_reward": 0.0, "step": 1267 }, { "clip_ratio": 0.0, "completion_length": 694.3750305175781, "epoch": 0.8453333333333334, "grad_norm": 1.4255295991897583, "kl": 1.482421875, "learning_rate": 2.1335100849183952e-07, "loss": 0.0678, "reward": 0.0833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 1268 }, { "clip_ratio": 0.0, "completion_length": 613.0416793823242, "epoch": 0.846, "grad_norm": 1.19693922996521, "kl": 1.02197265625, "learning_rate": 2.1156015460400337e-07, "loss": 0.0846, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 1269 }, { "clip_ratio": 0.0, "completion_length": 565.7083511352539, "epoch": 0.8466666666666667, "grad_norm": 0.4497241973876953, "kl": 0.575439453125, "learning_rate": 2.0977627815695215e-07, "loss": 0.1148, "reward": 0.3333333395421505, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.3333333395421505, "rewards/format_reward": 0.0, "step": 1270 }, { "clip_ratio": 0.0, "completion_length": 596.5000267028809, "epoch": 0.8473333333333334, "grad_norm": 0.5032573342323303, "kl": 1.7174072265625, "learning_rate": 2.0799938881112242e-07, "loss": 0.1692, "reward": 0.4583333358168602, "reward_std": 0.3410547897219658, "rewards/accuracy_reward": 0.4583333358168602, "rewards/format_reward": 0.0, "step": 1271 }, { "clip_ratio": 0.0, "completion_length": 588.2500305175781, "epoch": 0.848, "grad_norm": 1.5221086740493774, "kl": 0.416900634765625, "learning_rate": 2.062294961891138e-07, "loss": -0.007, "reward": 0.4583333544433117, "reward_std": 0.3061862289905548, "rewards/accuracy_reward": 0.4583333544433117, "rewards/format_reward": 0.0, "step": 1272 }, { "clip_ratio": 0.0, "completion_length": 461.91668701171875, "epoch": 0.8486666666666667, "grad_norm": 1.8427236080169678, "kl": 1.22216796875, "learning_rate": 2.0446660987563554e-07, "loss": 0.294, "reward": 0.5833333395421505, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.5833333395421505, "rewards/format_reward": 0.0, "step": 1273 }, { "clip_ratio": 0.0, "completion_length": 658.9583435058594, "epoch": 0.8493333333333334, "grad_norm": 1.1480318307876587, "kl": 1.0732421875, "learning_rate": 2.0271073941745365e-07, "loss": 0.1018, "reward": 0.3333333432674408, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 1274 }, { "clip_ratio": 0.0, "completion_length": 534.1250152587891, "epoch": 0.85, "grad_norm": 0.46283307671546936, "kl": 1.5550537109375, "learning_rate": 2.0096189432334195e-07, "loss": 0.2491, "reward": 0.4583333395421505, "reward_std": 0.5050228163599968, "rewards/accuracy_reward": 0.4583333395421505, "rewards/format_reward": 0.0, "step": 1275 }, { "clip_ratio": 0.0, "completion_length": 570.0000076293945, "epoch": 0.8506666666666667, "grad_norm": 0.9250903725624084, "kl": 0.96917724609375, "learning_rate": 1.992200840640284e-07, "loss": 0.048, "reward": 0.2916666716337204, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.2916666716337204, "rewards/format_reward": 0.0, "step": 1276 }, { "clip_ratio": 0.0, "completion_length": 379.3333435058594, "epoch": 0.8513333333333334, "grad_norm": 0.5942522287368774, "kl": 0.6893310546875, "learning_rate": 1.9748531807214358e-07, "loss": 0.1729, "reward": 0.7083333395421505, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.7083333395421505, "rewards/format_reward": 0.0, "step": 1277 }, { "clip_ratio": 0.0, "completion_length": 628.4166793823242, "epoch": 0.852, "grad_norm": 0.8198148012161255, "kl": 1.8851318359375, "learning_rate": 1.957576057421715e-07, "loss": 0.0913, "reward": 0.2916666753590107, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.2916666753590107, "rewards/format_reward": 0.0, "step": 1278 }, { "clip_ratio": 0.0, "completion_length": 670.9166870117188, "epoch": 0.8526666666666667, "grad_norm": 1.7241069078445435, "kl": 1.018310546875, "learning_rate": 1.9403695643039738e-07, "loss": 0.0916, "reward": 0.25, "reward_std": 0.273861289024353, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "step": 1279 }, { "clip_ratio": 0.0, "completion_length": 510.58334732055664, "epoch": 0.8533333333333334, "grad_norm": 0.8911008834838867, "kl": 0.685302734375, "learning_rate": 1.9232337945485655e-07, "loss": 0.1088, "reward": 0.3333333469927311, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.3333333469927311, "rewards/format_reward": 0.0, "step": 1280 }, { "clip_ratio": 0.0, "completion_length": 456.79168701171875, "epoch": 0.854, "grad_norm": 0.5442025661468506, "kl": 0.773193359375, "learning_rate": 1.9061688409528572e-07, "loss": 0.0429, "reward": 0.5416666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.5416666679084301, "rewards/format_reward": 0.0, "step": 1281 }, { "clip_ratio": 0.0, "completion_length": 501.79168701171875, "epoch": 0.8546666666666667, "grad_norm": 1.6180533170700073, "kl": 0.7886962890625, "learning_rate": 1.889174795930705e-07, "loss": 0.2176, "reward": 0.5000000186264515, "reward_std": 0.46232305467128754, "rewards/accuracy_reward": 0.5000000186264515, "rewards/format_reward": 0.0, "step": 1282 }, { "clip_ratio": 0.0, "completion_length": 718.0000152587891, "epoch": 0.8553333333333333, "grad_norm": 3.322932004928589, "kl": 1.57421875, "learning_rate": 1.8722517515119825e-07, "loss": 0.1452, "reward": 0.1666666679084301, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 1283 }, { "clip_ratio": 0.0, "completion_length": 588.1666717529297, "epoch": 0.856, "grad_norm": 0.8791308403015137, "kl": 0.6044921875, "learning_rate": 1.8553997993420495e-07, "loss": 0.0213, "reward": 0.125, "reward_std": 0.1369306445121765, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "step": 1284 }, { "clip_ratio": 0.0, "completion_length": 736.7500152587891, "epoch": 0.8566666666666667, "grad_norm": 0.5445953607559204, "kl": 0.5496826171875, "learning_rate": 1.838619030681269e-07, "loss": 0.053, "reward": 0.0833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 1285 }, { "clip_ratio": 0.0, "completion_length": 658.1250152587891, "epoch": 0.8573333333333333, "grad_norm": 1.0715062618255615, "kl": 1.208984375, "learning_rate": 1.8219095364045395e-07, "loss": 0.1605, "reward": 0.3333333358168602, "reward_std": 0.4779854416847229, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 1286 }, { "clip_ratio": 0.0, "completion_length": 547.1250152587891, "epoch": 0.858, "grad_norm": 0.6057612299919128, "kl": 0.853515625, "learning_rate": 1.8052714070007547e-07, "loss": 0.0264, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 1287 }, { "clip_ratio": 0.0, "completion_length": 684.6250305175781, "epoch": 0.8586666666666667, "grad_norm": 1.226332187652588, "kl": 1.20703125, "learning_rate": 1.7887047325723433e-07, "loss": 0.2747, "reward": 0.3333333358168602, "reward_std": 0.4779854342341423, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 1288 }, { "clip_ratio": 0.0, "completion_length": 568.0416870117188, "epoch": 0.8593333333333333, "grad_norm": 0.7299368381500244, "kl": 1.232421875, "learning_rate": 1.77220960283478e-07, "loss": 0.21, "reward": 0.3333333358168602, "reward_std": 0.40296074748039246, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 1289 }, { "clip_ratio": 0.0, "completion_length": 599.2916717529297, "epoch": 0.86, "grad_norm": 2.269897937774658, "kl": 1.509765625, "learning_rate": 1.7557861071160953e-07, "loss": 0.1769, "reward": 0.583333358168602, "reward_std": 0.46232305467128754, "rewards/accuracy_reward": 0.583333358168602, "rewards/format_reward": 0.0, "step": 1290 }, { "clip_ratio": 0.0, "completion_length": 508.29168701171875, "epoch": 0.8606666666666667, "grad_norm": 1.0078495740890503, "kl": 1.62347412109375, "learning_rate": 1.7394343343563907e-07, "loss": 0.2534, "reward": 0.5, "reward_std": 0.273861289024353, "rewards/accuracy_reward": 0.5, "rewards/format_reward": 0.0, "step": 1291 }, { "clip_ratio": 0.0, "completion_length": 535.5416870117188, "epoch": 0.8613333333333333, "grad_norm": 0.4500148594379425, "kl": 0.9541015625, "learning_rate": 1.7231543731073506e-07, "loss": 0.0363, "reward": 0.3333333469927311, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.3333333469927311, "rewards/format_reward": 0.0, "step": 1292 }, { "clip_ratio": 0.0, "completion_length": 676.8333435058594, "epoch": 0.862, "grad_norm": 9.850309371948242, "kl": 0.7529296875, "learning_rate": 1.706946311531779e-07, "loss": 0.069, "reward": 0.4166666679084301, "reward_std": 0.5128540024161339, "rewards/accuracy_reward": 0.4166666679084301, "rewards/format_reward": 0.0, "step": 1293 }, { "clip_ratio": 0.0, "completion_length": 663.2500228881836, "epoch": 0.8626666666666667, "grad_norm": 0.733267068862915, "kl": 1.5947265625, "learning_rate": 1.6908102374031125e-07, "loss": 0.1771, "reward": 0.3750000111758709, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 1294 }, { "clip_ratio": 0.0, "completion_length": 752.9166870117188, "epoch": 0.8633333333333333, "grad_norm": 0.9907534718513489, "kl": 1.57470703125, "learning_rate": 1.6747462381049416e-07, "loss": 0.1448, "reward": 0.1250000037252903, "reward_std": 0.306186206638813, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 1295 }, { "clip_ratio": 0.0, "completion_length": 604.5000152587891, "epoch": 0.864, "grad_norm": 2.1032540798187256, "kl": 2.060546875, "learning_rate": 1.6587544006305372e-07, "loss": 0.1533, "reward": 0.3333333432674408, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 1296 }, { "clip_ratio": 0.0, "completion_length": 579.2083587646484, "epoch": 0.8646666666666667, "grad_norm": 0.6150777339935303, "kl": 1.0234375, "learning_rate": 1.6428348115823977e-07, "loss": 0.0819, "reward": 0.291666679084301, "reward_std": 0.306186206638813, "rewards/accuracy_reward": 0.291666679084301, "rewards/format_reward": 0.0, "step": 1297 }, { "clip_ratio": 0.0, "completion_length": 618.1666870117188, "epoch": 0.8653333333333333, "grad_norm": 0.5496323704719543, "kl": 0.3984375, "learning_rate": 1.6269875571717528e-07, "loss": 0.0173, "reward": 0.3750000111758709, "reward_std": 0.3410547822713852, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 1298 }, { "clip_ratio": 0.0, "completion_length": 529.4166793823242, "epoch": 0.866, "grad_norm": 0.7664525508880615, "kl": 1.6171875, "learning_rate": 1.6112127232181162e-07, "loss": 0.1945, "reward": 0.3750000111758709, "reward_std": 0.3410547822713852, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 1299 }, { "clip_ratio": 0.0, "completion_length": 587.1666717529297, "epoch": 0.8666666666666667, "grad_norm": 1.1042970418930054, "kl": 1.77880859375, "learning_rate": 1.5955103951488177e-07, "loss": 0.225, "reward": 0.3333333432674408, "reward_std": 0.46232306212186813, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 1300 }, { "clip_ratio": 0.0, "completion_length": 650.5416793823242, "epoch": 0.8673333333333333, "grad_norm": 0.5317118763923645, "kl": 1.52734375, "learning_rate": 1.5798806579985247e-07, "loss": 0.1877, "reward": 0.3750000111758709, "reward_std": 0.4893604516983032, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 1301 }, { "clip_ratio": 0.0, "completion_length": 575.8750152587891, "epoch": 0.868, "grad_norm": 1.00601065158844, "kl": 1.3985595703125, "learning_rate": 1.5643235964088065e-07, "loss": 0.1022, "reward": 0.4166666716337204, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.4166666716337204, "rewards/format_reward": 0.0, "step": 1302 }, { "clip_ratio": 0.0, "completion_length": 565.4583587646484, "epoch": 0.8686666666666667, "grad_norm": 0.5771857500076294, "kl": 1.654541015625, "learning_rate": 1.548839294627663e-07, "loss": 0.1301, "reward": 0.2500000074505806, "reward_std": 0.3332236111164093, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 1303 }, { "clip_ratio": 0.0, "completion_length": 526.0833511352539, "epoch": 0.8693333333333333, "grad_norm": 1.0279359817504883, "kl": 0.348907470703125, "learning_rate": 1.53342783650906e-07, "loss": 0.0391, "reward": 0.4583333395421505, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.4583333395421505, "rewards/format_reward": 0.0, "step": 1304 }, { "clip_ratio": 0.0, "completion_length": 402.95834732055664, "epoch": 0.87, "grad_norm": 0.9338709712028503, "kl": 0.76904296875, "learning_rate": 1.5180893055124977e-07, "loss": 0.1252, "reward": 0.291666679084301, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.291666679084301, "rewards/format_reward": 0.0, "step": 1305 }, { "clip_ratio": 0.0, "completion_length": 593.7500228881836, "epoch": 0.8706666666666667, "grad_norm": 2.9501516819000244, "kl": 1.401611328125, "learning_rate": 1.5028237847025307e-07, "loss": 0.2375, "reward": 0.416666679084301, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.416666679084301, "rewards/format_reward": 0.0, "step": 1306 }, { "clip_ratio": 0.0, "completion_length": 462.7083511352539, "epoch": 0.8713333333333333, "grad_norm": 1.413503885269165, "kl": 0.8077392578125, "learning_rate": 1.4876313567483446e-07, "loss": 0.2161, "reward": 0.6666666828095913, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.6666666828095913, "rewards/format_reward": 0.0, "step": 1307 }, { "clip_ratio": 0.0, "completion_length": 592.2500267028809, "epoch": 0.872, "grad_norm": 1.578917145729065, "kl": 2.9267578125, "learning_rate": 1.4725121039232948e-07, "loss": 0.2202, "reward": 0.2500000074505806, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 1308 }, { "clip_ratio": 0.0, "completion_length": 312.45833587646484, "epoch": 0.8726666666666667, "grad_norm": 1.218940258026123, "kl": 0.27825927734375, "learning_rate": 1.457466108104456e-07, "loss": -0.0229, "reward": 0.6250000149011612, "reward_std": 0.3602609932422638, "rewards/accuracy_reward": 0.6250000149011612, "rewards/format_reward": 0.0, "step": 1309 }, { "clip_ratio": 0.0, "completion_length": 637.6250305175781, "epoch": 0.8733333333333333, "grad_norm": 1.2396628856658936, "kl": 1.3193359375, "learning_rate": 1.4424934507721927e-07, "loss": 0.2633, "reward": 0.4166666716337204, "reward_std": 0.5320602059364319, "rewards/accuracy_reward": 0.4166666716337204, "rewards/format_reward": 0.0, "step": 1310 }, { "clip_ratio": 0.0, "completion_length": 671.5000305175781, "epoch": 0.874, "grad_norm": 0.5698392987251282, "kl": 1.5732421875, "learning_rate": 1.4275942130097098e-07, "loss": 0.1351, "reward": 0.2916666716337204, "reward_std": 0.39512956142425537, "rewards/accuracy_reward": 0.2916666716337204, "rewards/format_reward": 0.0, "step": 1311 }, { "clip_ratio": 0.0, "completion_length": 502.16668701171875, "epoch": 0.8746666666666667, "grad_norm": 0.7037941217422485, "kl": 0.96734619140625, "learning_rate": 1.4127684755026072e-07, "loss": 0.1233, "reward": 0.3750000149011612, "reward_std": 0.43528566509485245, "rewards/accuracy_reward": 0.3750000149011612, "rewards/format_reward": 0.0, "step": 1312 }, { "clip_ratio": 0.0, "completion_length": 665.3333511352539, "epoch": 0.8753333333333333, "grad_norm": 3.463456630706787, "kl": 1.258026123046875, "learning_rate": 1.3980163185384599e-07, "loss": 0.1867, "reward": 0.2500000074505806, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 1313 }, { "clip_ratio": 0.0, "completion_length": 809.8333435058594, "epoch": 0.876, "grad_norm": 6.841768264770508, "kl": 1.796875, "learning_rate": 1.3833378220063715e-07, "loss": 0.1089, "reward": 0.0833333358168602, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 1314 }, { "clip_ratio": 0.0, "completion_length": 592.7500228881836, "epoch": 0.8766666666666667, "grad_norm": 1.8474562168121338, "kl": 1.80029296875, "learning_rate": 1.3687330653965362e-07, "loss": 0.1508, "reward": 0.2916666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.2916666679084301, "rewards/format_reward": 0.0, "step": 1315 }, { "clip_ratio": 0.0, "completion_length": 478.29168701171875, "epoch": 0.8773333333333333, "grad_norm": 0.9747337102890015, "kl": 1.37518310546875, "learning_rate": 1.354202127799825e-07, "loss": 0.1409, "reward": 0.4583333395421505, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.4583333395421505, "rewards/format_reward": 0.0, "step": 1316 }, { "clip_ratio": 0.0, "completion_length": 749.5416946411133, "epoch": 0.878, "grad_norm": 1.304632544517517, "kl": 1.4677734375, "learning_rate": 1.3397450879073447e-07, "loss": 0.081, "reward": 0.2083333395421505, "reward_std": 0.3602609932422638, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 1317 }, { "clip_ratio": 0.0, "completion_length": 674.2500076293945, "epoch": 0.8786666666666667, "grad_norm": 1.0042383670806885, "kl": 2.490234375, "learning_rate": 1.3253620240100105e-07, "loss": 0.2007, "reward": 0.416666679084301, "reward_std": 0.44311685115098953, "rewards/accuracy_reward": 0.416666679084301, "rewards/format_reward": 0.0, "step": 1318 }, { "clip_ratio": 0.0, "completion_length": 610.8750152587891, "epoch": 0.8793333333333333, "grad_norm": 0.31455183029174805, "kl": 0.359375, "learning_rate": 1.3110530139981352e-07, "loss": -0.0048, "reward": 0.3750000149011612, "reward_std": 0.3602609932422638, "rewards/accuracy_reward": 0.3750000149011612, "rewards/format_reward": 0.0, "step": 1319 }, { "clip_ratio": 0.0, "completion_length": 703.3750305175781, "epoch": 0.88, "grad_norm": 0.9506360292434692, "kl": 0.36669921875, "learning_rate": 1.2968181353609853e-07, "loss": 0.0174, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 1320 }, { "clip_ratio": 0.0, "completion_length": 371.4166831970215, "epoch": 0.8806666666666667, "grad_norm": 1.0033214092254639, "kl": 2.2105712890625, "learning_rate": 1.282657465186397e-07, "loss": 0.092, "reward": 0.5833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.5833333358168602, "rewards/format_reward": 0.0, "step": 1321 }, { "clip_ratio": 0.0, "completion_length": 534.2916793823242, "epoch": 0.8813333333333333, "grad_norm": 1.2356717586517334, "kl": 1.365234375, "learning_rate": 1.2685710801603145e-07, "loss": 0.1047, "reward": 0.4166666679084301, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.4166666679084301, "rewards/format_reward": 0.0, "step": 1322 }, { "clip_ratio": 0.0, "completion_length": 568.2083511352539, "epoch": 0.882, "grad_norm": 2.2706782817840576, "kl": 2.5302734375, "learning_rate": 1.2545590565664057e-07, "loss": 0.1564, "reward": 0.291666679084301, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.291666679084301, "rewards/format_reward": 0.0, "step": 1323 }, { "clip_ratio": 0.0, "completion_length": 573.0833587646484, "epoch": 0.8826666666666667, "grad_norm": 1.3609440326690674, "kl": 2.595703125, "learning_rate": 1.240621470285645e-07, "loss": 0.3694, "reward": 0.2916666753590107, "reward_std": 0.3602609932422638, "rewards/accuracy_reward": 0.2916666753590107, "rewards/format_reward": 0.0, "step": 1324 }, { "clip_ratio": 0.0, "completion_length": 634.5833587646484, "epoch": 0.8833333333333333, "grad_norm": 0.42868348956108093, "kl": 0.9359130859375, "learning_rate": 1.2267583967958918e-07, "loss": 0.1373, "reward": 0.2916666679084301, "reward_std": 0.37592335790395737, "rewards/accuracy_reward": 0.2916666679084301, "rewards/format_reward": 0.0, "step": 1325 }, { "clip_ratio": 0.0, "completion_length": 641.0833511352539, "epoch": 0.884, "grad_norm": 53.6770133972168, "kl": 8.4970703125, "learning_rate": 1.2129699111714841e-07, "loss": 0.4699, "reward": 0.2083333358168602, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.2083333358168602, "rewards/format_reward": 0.0, "step": 1326 }, { "clip_ratio": 0.0, "completion_length": 647.5000152587891, "epoch": 0.8846666666666667, "grad_norm": 1.7472645044326782, "kl": 1.5302734375, "learning_rate": 1.199256088082844e-07, "loss": 0.1506, "reward": 0.1250000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 1327 }, { "clip_ratio": 0.0, "completion_length": 415.25000762939453, "epoch": 0.8853333333333333, "grad_norm": 0.6521641612052917, "kl": 0.104736328125, "learning_rate": 1.1856170017960611e-07, "loss": -0.0024, "reward": 0.6250000149011612, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.6250000149011612, "rewards/format_reward": 0.0, "step": 1328 }, { "clip_ratio": 0.0, "completion_length": 600.0000076293945, "epoch": 0.886, "grad_norm": 6.923852920532227, "kl": 1.2242431640625, "learning_rate": 1.1720527261724939e-07, "loss": 0.1007, "reward": 0.4583333432674408, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.4583333432674408, "rewards/format_reward": 0.0, "step": 1329 }, { "clip_ratio": 0.0, "completion_length": 607.4583435058594, "epoch": 0.8866666666666667, "grad_norm": 1.9614558219909668, "kl": 1.75927734375, "learning_rate": 1.1585633346683655e-07, "loss": 0.1649, "reward": 0.5000000111758709, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.5000000111758709, "rewards/format_reward": 0.0, "step": 1330 }, { "clip_ratio": 0.0, "completion_length": 665.2083435058594, "epoch": 0.8873333333333333, "grad_norm": 2.9625587463378906, "kl": 1.44921875, "learning_rate": 1.1451489003343762e-07, "loss": 0.1919, "reward": 0.3333333395421505, "reward_std": 0.3680921792984009, "rewards/accuracy_reward": 0.3333333395421505, "rewards/format_reward": 0.0, "step": 1331 }, { "clip_ratio": 0.0, "completion_length": 641.3333511352539, "epoch": 0.888, "grad_norm": 1.5786848068237305, "kl": 2.0654296875, "learning_rate": 1.1318094958153047e-07, "loss": 0.0954, "reward": 0.3333333358168602, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 1332 }, { "clip_ratio": 0.0, "completion_length": 561.3333549499512, "epoch": 0.8886666666666667, "grad_norm": 2.9198813438415527, "kl": 1.694091796875, "learning_rate": 1.1185451933495994e-07, "loss": 0.117, "reward": 0.4583333358168602, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.4583333358168602, "rewards/format_reward": 0.0, "step": 1333 }, { "clip_ratio": 0.0, "completion_length": 783.7916793823242, "epoch": 0.8893333333333333, "grad_norm": 0.5971931219100952, "kl": 1.6953125, "learning_rate": 1.1053560647690109e-07, "loss": 0.1911, "reward": 0.1666666716337204, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 1334 }, { "clip_ratio": 0.0, "completion_length": 668.1250152587891, "epoch": 0.89, "grad_norm": 2.052192449569702, "kl": 0.80078125, "learning_rate": 1.0922421814981903e-07, "loss": 0.095, "reward": 0.2083333395421505, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 1335 }, { "clip_ratio": 0.0, "completion_length": 605.9166870117188, "epoch": 0.8906666666666667, "grad_norm": 0.6041466593742371, "kl": 1.619140625, "learning_rate": 1.0792036145542966e-07, "loss": 0.1195, "reward": 0.2083333395421505, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 1336 }, { "clip_ratio": 0.0, "completion_length": 421.87501525878906, "epoch": 0.8913333333333333, "grad_norm": 0.5697200894355774, "kl": 1.2540283203125, "learning_rate": 1.0662404345466281e-07, "loss": 0.1821, "reward": 0.375, "reward_std": 0.1369306445121765, "rewards/accuracy_reward": 0.375, "rewards/format_reward": 0.0, "step": 1337 }, { "clip_ratio": 0.0, "completion_length": 409.87500762939453, "epoch": 0.892, "grad_norm": 2.6190624237060547, "kl": 1.857421875, "learning_rate": 1.0533527116762298e-07, "loss": 0.1767, "reward": 0.6250000223517418, "reward_std": 0.4701542407274246, "rewards/accuracy_reward": 0.6250000223517418, "rewards/format_reward": 0.0, "step": 1338 }, { "clip_ratio": 0.0, "completion_length": 551.5, "epoch": 0.8926666666666667, "grad_norm": 0.9679128527641296, "kl": 1.048828125, "learning_rate": 1.0405405157355069e-07, "loss": 0.1529, "reward": 0.2500000111758709, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.2500000111758709, "rewards/format_reward": 0.0, "step": 1339 }, { "clip_ratio": 0.0, "completion_length": 687.0000228881836, "epoch": 0.8933333333333333, "grad_norm": 1.5186926126480103, "kl": 1.04052734375, "learning_rate": 1.0278039161078634e-07, "loss": 0.1652, "reward": 0.3333333432674408, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 1340 }, { "clip_ratio": 0.0, "completion_length": 523.1250228881836, "epoch": 0.894, "grad_norm": 0.5501824617385864, "kl": 0.6658935546875, "learning_rate": 1.0151429817673174e-07, "loss": 0.0718, "reward": 0.3333333469927311, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.3333333469927311, "rewards/format_reward": 0.0, "step": 1341 }, { "clip_ratio": 0.0, "completion_length": 730.6666870117188, "epoch": 0.8946666666666667, "grad_norm": 1.4511724710464478, "kl": 0.705322265625, "learning_rate": 1.00255778127812e-07, "loss": 0.0988, "reward": 0.2500000111758709, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.2500000111758709, "rewards/format_reward": 0.0, "step": 1342 }, { "clip_ratio": 0.0, "completion_length": 748.7083435058594, "epoch": 0.8953333333333333, "grad_norm": 1.058302879333496, "kl": 1.109619140625, "learning_rate": 9.900483827944018e-08, "loss": 0.047, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 1343 }, { "clip_ratio": 0.0, "completion_length": 551.7500152587891, "epoch": 0.896, "grad_norm": 0.4251255691051483, "kl": 0.651123046875, "learning_rate": 9.776148540597835e-08, "loss": 0.0454, "reward": 0.1666666716337204, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 1344 }, { "clip_ratio": 0.0, "completion_length": 663.8333435058594, "epoch": 0.8966666666666666, "grad_norm": 1.0505051612854004, "kl": 1.5029296875, "learning_rate": 9.652572624070294e-08, "loss": 0.2127, "reward": 0.3750000149011612, "reward_std": 0.3602609932422638, "rewards/accuracy_reward": 0.3750000149011612, "rewards/format_reward": 0.0, "step": 1345 }, { "clip_ratio": 0.0, "completion_length": 738.0000076293945, "epoch": 0.8973333333333333, "grad_norm": 1.343625783920288, "kl": 1.4462890625, "learning_rate": 9.529756747576663e-08, "loss": 0.121, "reward": 0.2500000111758709, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.2500000111758709, "rewards/format_reward": 0.0, "step": 1346 }, { "clip_ratio": 0.0, "completion_length": 587.3750152587891, "epoch": 0.898, "grad_norm": 1.9155170917510986, "kl": 1.5078125, "learning_rate": 9.407701576216282e-08, "loss": 0.0715, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 1347 }, { "clip_ratio": 0.0, "completion_length": 646.9166831970215, "epoch": 0.8986666666666666, "grad_norm": 0.3998793363571167, "kl": 1.70562744140625, "learning_rate": 9.286407770968975e-08, "loss": 0.1354, "reward": 0.3333333469927311, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.3333333469927311, "rewards/format_reward": 0.0, "step": 1348 }, { "clip_ratio": 0.0, "completion_length": 577.6250076293945, "epoch": 0.8993333333333333, "grad_norm": 1.2635958194732666, "kl": 0.488037109375, "learning_rate": 9.165875988691474e-08, "loss": 0.0522, "reward": 0.5000000111758709, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.5000000111758709, "rewards/format_reward": 0.0, "step": 1349 }, { "clip_ratio": 0.0, "completion_length": 486.7916793823242, "epoch": 0.9, "grad_norm": 0.6671023368835449, "kl": 0.77606201171875, "learning_rate": 9.046106882113752e-08, "loss": 0.0081, "reward": 0.3333333358168602, "reward_std": 0.20412415266036987, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 1350 }, { "clip_ratio": 0.0, "completion_length": 574.5416870117188, "epoch": 0.9006666666666666, "grad_norm": 1.8972612619400024, "kl": 1.0093994140625, "learning_rate": 8.92710109983565e-08, "loss": 0.0489, "reward": 0.2083333395421505, "reward_std": 0.3602609932422638, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 1351 }, { "clip_ratio": 0.0, "completion_length": 601.9166870117188, "epoch": 0.9013333333333333, "grad_norm": 1.6796592473983765, "kl": 1.288818359375, "learning_rate": 8.808859286323279e-08, "loss": 0.193, "reward": 0.3750000074505806, "reward_std": 0.39512956142425537, "rewards/accuracy_reward": 0.3750000074505806, "rewards/format_reward": 0.0, "step": 1352 }, { "clip_ratio": 0.0, "completion_length": 782.1666870117188, "epoch": 0.902, "grad_norm": 2.9987287521362305, "kl": 3.13818359375, "learning_rate": 8.691382081905496e-08, "loss": 0.1861, "reward": 0.0416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 1353 }, { "clip_ratio": 0.0, "completion_length": 715.7083435058594, "epoch": 0.9026666666666666, "grad_norm": 1.3511848449707031, "kl": 2.56640625, "learning_rate": 8.574670122770523e-08, "loss": 0.287, "reward": 0.2500000074505806, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 1354 }, { "clip_ratio": 0.0, "completion_length": 512.2500228881836, "epoch": 0.9033333333333333, "grad_norm": 0.6283645629882812, "kl": 0.63232421875, "learning_rate": 8.458724040962473e-08, "loss": 0.0979, "reward": 0.2083333432674408, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.2083333432674408, "rewards/format_reward": 0.0, "step": 1355 }, { "clip_ratio": 0.0, "completion_length": 490.0416946411133, "epoch": 0.904, "grad_norm": 1.4534605741500854, "kl": 0.591064453125, "learning_rate": 8.34354446437785e-08, "loss": 0.1019, "reward": 0.6250000111758709, "reward_std": 0.3410547822713852, "rewards/accuracy_reward": 0.6250000111758709, "rewards/format_reward": 0.0, "step": 1356 }, { "clip_ratio": 0.0, "completion_length": 525.4166793823242, "epoch": 0.9046666666666666, "grad_norm": 0.8719340562820435, "kl": 0.77447509765625, "learning_rate": 8.229132016762275e-08, "loss": 0.0696, "reward": 0.2916666753590107, "reward_std": 0.3602609932422638, "rewards/accuracy_reward": 0.2916666753590107, "rewards/format_reward": 0.0, "step": 1357 }, { "clip_ratio": 0.0, "completion_length": 517.6666870117188, "epoch": 0.9053333333333333, "grad_norm": 0.8715060353279114, "kl": 1.699951171875, "learning_rate": 8.115487317706944e-08, "loss": 0.1784, "reward": 0.2500000074505806, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 1358 }, { "clip_ratio": 0.0, "completion_length": 587.8750152587891, "epoch": 0.906, "grad_norm": 1.2465791702270508, "kl": 1.77752685546875, "learning_rate": 8.002610982645558e-08, "loss": 0.1719, "reward": 0.2500000037252903, "reward_std": 0.3680921792984009, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.0, "step": 1359 }, { "clip_ratio": 0.0, "completion_length": 581.5416717529297, "epoch": 0.9066666666666666, "grad_norm": 0.5284703373908997, "kl": 1.03076171875, "learning_rate": 7.89050362285062e-08, "loss": 0.1312, "reward": 0.2916666679084301, "reward_std": 0.37592336535453796, "rewards/accuracy_reward": 0.2916666679084301, "rewards/format_reward": 0.0, "step": 1360 }, { "clip_ratio": 0.0, "completion_length": 484.91668701171875, "epoch": 0.9073333333333333, "grad_norm": 1.7086902856826782, "kl": 1.2099609375, "learning_rate": 7.779165845430376e-08, "loss": 0.1006, "reward": 0.3333333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 1361 }, { "clip_ratio": 0.0, "completion_length": 492.12500762939453, "epoch": 0.908, "grad_norm": 1.5797888040542603, "kl": 1.403076171875, "learning_rate": 7.668598253325521e-08, "loss": 0.1334, "reward": 0.6666666828095913, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.6666666828095913, "rewards/format_reward": 0.0, "step": 1362 }, { "clip_ratio": 0.0, "completion_length": 699.3333435058594, "epoch": 0.9086666666666666, "grad_norm": 2.1805591583251953, "kl": 1.9560546875, "learning_rate": 7.558801445305791e-08, "loss": 0.0847, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 1363 }, { "clip_ratio": 0.0, "completion_length": 492.54168701171875, "epoch": 0.9093333333333333, "grad_norm": 1.4022495746612549, "kl": 1.3525390625, "learning_rate": 7.449776015966825e-08, "loss": 0.1955, "reward": 0.5000000149011612, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.5000000149011612, "rewards/format_reward": 0.0, "step": 1364 }, { "clip_ratio": 0.0, "completion_length": 527.9166870117188, "epoch": 0.91, "grad_norm": 1.5445491075515747, "kl": 1.76171875, "learning_rate": 7.341522555726971e-08, "loss": 0.1763, "reward": 0.125, "reward_std": 0.1369306445121765, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "step": 1365 }, { "clip_ratio": 0.0, "completion_length": 569.7500076293945, "epoch": 0.9106666666666666, "grad_norm": 1.2069586515426636, "kl": 0.8941650390625, "learning_rate": 7.234041650823953e-08, "loss": -0.0115, "reward": 0.3333333395421505, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.3333333395421505, "rewards/format_reward": 0.0, "step": 1366 }, { "clip_ratio": 0.0, "completion_length": 594.0416793823242, "epoch": 0.9113333333333333, "grad_norm": 0.6288086175918579, "kl": 0.86083984375, "learning_rate": 7.12733388331191e-08, "loss": 0.0032, "reward": 0.2500000111758709, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.2500000111758709, "rewards/format_reward": 0.0, "step": 1367 }, { "clip_ratio": 0.0, "completion_length": 588.0833587646484, "epoch": 0.912, "grad_norm": 2.2703778743743896, "kl": 1.05908203125, "learning_rate": 7.021399831057961e-08, "loss": 0.2023, "reward": 0.5000000149011612, "reward_std": 0.4971916303038597, "rewards/accuracy_reward": 0.5000000149011612, "rewards/format_reward": 0.0, "step": 1368 }, { "clip_ratio": 0.0, "completion_length": 422.29168701171875, "epoch": 0.9126666666666666, "grad_norm": 1.8955916166305542, "kl": 1.2752685546875, "learning_rate": 6.916240067739349e-08, "loss": 0.1444, "reward": 0.3750000111758709, "reward_std": 0.3410547897219658, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 1369 }, { "clip_ratio": 0.0, "completion_length": 612.2916946411133, "epoch": 0.9133333333333333, "grad_norm": 0.8215115666389465, "kl": 0.460540771484375, "learning_rate": 6.811855162840214e-08, "loss": 0.0237, "reward": 0.3333333358168602, "reward_std": 0.40296074748039246, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 1370 }, { "clip_ratio": 0.0, "completion_length": 691.9583511352539, "epoch": 0.914, "grad_norm": 1.0539863109588623, "kl": 1.785400390625, "learning_rate": 6.708245681648456e-08, "loss": 0.1107, "reward": 0.2500000074505806, "reward_std": 0.3332236111164093, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 1371 }, { "clip_ratio": 0.0, "completion_length": 617.1250152587891, "epoch": 0.9146666666666666, "grad_norm": 0.7596750259399414, "kl": 1.365234375, "learning_rate": 6.605412185252751e-08, "loss": 0.1452, "reward": 0.1250000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 1372 }, { "clip_ratio": 0.0, "completion_length": 556.7500152587891, "epoch": 0.9153333333333333, "grad_norm": 0.33255094289779663, "kl": 0.6259765625, "learning_rate": 6.503355230539553e-08, "loss": -0.0209, "reward": 0.0833333358168602, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 1373 }, { "clip_ratio": 0.0, "completion_length": 501.87501525878906, "epoch": 0.916, "grad_norm": 0.9830915331840515, "kl": 1.09375, "learning_rate": 6.402075370189914e-08, "loss": 0.2552, "reward": 0.2916666753590107, "reward_std": 0.4893604442477226, "rewards/accuracy_reward": 0.2916666753590107, "rewards/format_reward": 0.0, "step": 1374 }, { "clip_ratio": 0.0, "completion_length": 471.87501525878906, "epoch": 0.9166666666666666, "grad_norm": 0.846712589263916, "kl": 0.8924560546875, "learning_rate": 6.301573152676666e-08, "loss": 0.0896, "reward": 0.5416666716337204, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.5416666716337204, "rewards/format_reward": 0.0, "step": 1375 }, { "clip_ratio": 0.0, "completion_length": 531.6250228881836, "epoch": 0.9173333333333333, "grad_norm": 1.2041231393814087, "kl": 0.79443359375, "learning_rate": 6.201849122261377e-08, "loss": 0.1286, "reward": 0.4166666716337204, "reward_std": 0.40296074748039246, "rewards/accuracy_reward": 0.4166666716337204, "rewards/format_reward": 0.0, "step": 1376 }, { "clip_ratio": 0.0, "completion_length": 690.2083435058594, "epoch": 0.918, "grad_norm": 0.3309684693813324, "kl": 0.6075439453125, "learning_rate": 6.102903818991374e-08, "loss": 0.0956, "reward": 0.1666666679084301, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 1377 }, { "clip_ratio": 0.0, "completion_length": 601.5833587646484, "epoch": 0.9186666666666666, "grad_norm": 0.8770349025726318, "kl": 1.46484375, "learning_rate": 6.004737778696868e-08, "loss": 0.0984, "reward": 0.1666666679084301, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 1378 }, { "clip_ratio": 0.0, "completion_length": 545.7083511352539, "epoch": 0.9193333333333333, "grad_norm": 0.37358197569847107, "kl": 1.15673828125, "learning_rate": 5.9073515329880645e-08, "loss": 0.1118, "reward": 0.1666666716337204, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 1379 }, { "clip_ratio": 0.0, "completion_length": 367.5833435058594, "epoch": 0.92, "grad_norm": 0.4871121048927307, "kl": 1.410369873046875, "learning_rate": 5.810745609252166e-08, "loss": 0.1725, "reward": 0.625, "reward_std": 0.41079193353652954, "rewards/accuracy_reward": 0.625, "rewards/format_reward": 0.0, "step": 1380 }, { "clip_ratio": 0.0, "completion_length": 582.0416793823242, "epoch": 0.9206666666666666, "grad_norm": 1.064256191253662, "kl": 1.364013671875, "learning_rate": 5.7149205306507356e-08, "loss": 0.1299, "reward": 0.25, "reward_std": 0.273861289024353, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "step": 1381 }, { "clip_ratio": 0.0, "completion_length": 266.4166717529297, "epoch": 0.9213333333333333, "grad_norm": 0.4518880546092987, "kl": 0.22735595703125, "learning_rate": 5.6198768161166036e-08, "loss": 0.0808, "reward": 0.8750000149011612, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.8750000149011612, "rewards/format_reward": 0.0, "step": 1382 }, { "clip_ratio": 0.0, "completion_length": 668.8750152587891, "epoch": 0.922, "grad_norm": 0.38337698578834534, "kl": 0.352081298828125, "learning_rate": 5.5256149803512844e-08, "loss": 0.0606, "reward": 0.1666666679084301, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 1383 }, { "clip_ratio": 0.0, "completion_length": 319.7083435058594, "epoch": 0.9226666666666666, "grad_norm": 0.33102670311927795, "kl": 0.29107666015625, "learning_rate": 5.4321355338220465e-08, "loss": 0.0732, "reward": 0.7083333544433117, "reward_std": 0.306186206638813, "rewards/accuracy_reward": 0.7083333544433117, "rewards/format_reward": 0.0, "step": 1384 }, { "clip_ratio": 0.0, "completion_length": 595.1666793823242, "epoch": 0.9233333333333333, "grad_norm": 0.773875892162323, "kl": 1.08984375, "learning_rate": 5.339438982759165e-08, "loss": 0.0484, "reward": 0.1666666716337204, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 1385 }, { "clip_ratio": 0.0, "completion_length": 598.9583435058594, "epoch": 0.924, "grad_norm": 0.8809089064598083, "kl": 1.339599609375, "learning_rate": 5.247525829153288e-08, "loss": 0.0959, "reward": 0.1250000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 1386 }, { "clip_ratio": 0.0, "completion_length": 509.4166793823242, "epoch": 0.9246666666666666, "grad_norm": 0.7207773923873901, "kl": 0.66009521484375, "learning_rate": 5.1563965707525916e-08, "loss": 0.0425, "reward": 0.5416666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.5416666679084301, "rewards/format_reward": 0.0, "step": 1387 }, { "clip_ratio": 0.0, "completion_length": 488.2916717529297, "epoch": 0.9253333333333333, "grad_norm": 2.2440106868743896, "kl": 0.71246337890625, "learning_rate": 5.066051701060098e-08, "loss": 0.1257, "reward": 0.6250000149011612, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.6250000149011612, "rewards/format_reward": 0.0, "step": 1388 }, { "clip_ratio": 0.0, "completion_length": 585.3333587646484, "epoch": 0.926, "grad_norm": 0.6839295625686646, "kl": 1.5394287109375, "learning_rate": 4.9764917093310935e-08, "loss": 0.0498, "reward": 0.4166666828095913, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.4166666828095913, "rewards/format_reward": 0.0, "step": 1389 }, { "clip_ratio": 0.0, "completion_length": 695.0833511352539, "epoch": 0.9266666666666666, "grad_norm": 2.1217339038848877, "kl": 0.79248046875, "learning_rate": 4.887717080570431e-08, "loss": 0.1259, "reward": 0.291666679084301, "reward_std": 0.3061862140893936, "rewards/accuracy_reward": 0.291666679084301, "rewards/format_reward": 0.0, "step": 1390 }, { "clip_ratio": 0.0, "completion_length": 553.5833511352539, "epoch": 0.9273333333333333, "grad_norm": 0.841241717338562, "kl": 1.00250244140625, "learning_rate": 4.7997282955297986e-08, "loss": 0.0571, "reward": 0.1250000037252903, "reward_std": 0.3061862140893936, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 1391 }, { "clip_ratio": 0.0, "completion_length": 619.6666870117188, "epoch": 0.928, "grad_norm": 0.8053234219551086, "kl": 1.21875, "learning_rate": 4.712525830705339e-08, "loss": 0.1571, "reward": 0.2083333395421505, "reward_std": 0.3602609932422638, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 1392 }, { "clip_ratio": 0.0, "completion_length": 527.4583511352539, "epoch": 0.9286666666666666, "grad_norm": 0.5859879851341248, "kl": 1.495849609375, "learning_rate": 4.6261101583348344e-08, "loss": 0.1989, "reward": 0.4583333358168602, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.4583333358168602, "rewards/format_reward": 0.0, "step": 1393 }, { "clip_ratio": 0.0, "completion_length": 580.3333511352539, "epoch": 0.9293333333333333, "grad_norm": 0.8214588761329651, "kl": 0.72515869140625, "learning_rate": 4.540481746395359e-08, "loss": 0.1434, "reward": 0.4583333507180214, "reward_std": 0.3602609932422638, "rewards/accuracy_reward": 0.4583333507180214, "rewards/format_reward": 0.0, "step": 1394 }, { "clip_ratio": 0.0, "completion_length": 664.5416870117188, "epoch": 0.93, "grad_norm": 0.5648185610771179, "kl": 1.6396484375, "learning_rate": 4.455641058600529e-08, "loss": 0.1711, "reward": 0.1666666716337204, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 1395 }, { "clip_ratio": 0.0, "completion_length": 725.0416870117188, "epoch": 0.9306666666666666, "grad_norm": 0.7184426188468933, "kl": 0.61138916015625, "learning_rate": 4.371588554398193e-08, "loss": 0.0429, "reward": 0.2083333395421505, "reward_std": 0.43528566509485245, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 1396 }, { "clip_ratio": 0.0, "completion_length": 504.2083435058594, "epoch": 0.9313333333333333, "grad_norm": 1.4880552291870117, "kl": 0.8994140625, "learning_rate": 4.2883246889678264e-08, "loss": 0.1327, "reward": 0.2083333432674408, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.2083333432674408, "rewards/format_reward": 0.0, "step": 1397 }, { "clip_ratio": 0.0, "completion_length": 535.9583435058594, "epoch": 0.932, "grad_norm": 0.5331709384918213, "kl": 0.36993408203125, "learning_rate": 4.205849913218074e-08, "loss": 0.0161, "reward": 0.4583333432674408, "reward_std": 0.5242290198802948, "rewards/accuracy_reward": 0.4583333432674408, "rewards/format_reward": 0.0, "step": 1398 }, { "clip_ratio": 0.0, "completion_length": 497.2916717529297, "epoch": 0.9326666666666666, "grad_norm": 0.7155057191848755, "kl": 0.514892578125, "learning_rate": 4.124164673784348e-08, "loss": -0.0066, "reward": 0.2500000111758709, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.2500000111758709, "rewards/format_reward": 0.0, "step": 1399 }, { "clip_ratio": 0.0, "completion_length": 621.7500152587891, "epoch": 0.9333333333333333, "grad_norm": 1.1602201461791992, "kl": 0.98681640625, "learning_rate": 4.0432694130264294e-08, "loss": 0.0478, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 1400 }, { "clip_ratio": 0.0, "completion_length": 414.9166793823242, "epoch": 0.934, "grad_norm": 0.5087230801582336, "kl": 0.4859619140625, "learning_rate": 3.9631645690259566e-08, "loss": 0.0956, "reward": 0.4583333358168602, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.4583333358168602, "rewards/format_reward": 0.0, "step": 1401 }, { "clip_ratio": 0.0, "completion_length": 721.6667022705078, "epoch": 0.9346666666666666, "grad_norm": 0.9768966436386108, "kl": 1.8408203125, "learning_rate": 3.8838505755841915e-08, "loss": 0.1424, "reward": 0.0833333358168602, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 1402 }, { "clip_ratio": 0.0, "completion_length": 527.7083358764648, "epoch": 0.9353333333333333, "grad_norm": 1.0609781742095947, "kl": 1.773193359375, "learning_rate": 3.8053278622196045e-08, "loss": 0.1834, "reward": 0.2083333395421505, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 1403 }, { "clip_ratio": 0.0, "completion_length": 681.9166793823242, "epoch": 0.936, "grad_norm": 2.2927372455596924, "kl": 2.5205078125, "learning_rate": 3.72759685416551e-08, "loss": 0.1656, "reward": 0.125, "reward_std": 0.1369306445121765, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "step": 1404 }, { "clip_ratio": 0.0, "completion_length": 677.8333435058594, "epoch": 0.9366666666666666, "grad_norm": 0.9890009760856628, "kl": 1.317138671875, "learning_rate": 3.650657972367871e-08, "loss": 0.2987, "reward": 0.3333333432674408, "reward_std": 0.3872983753681183, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 1405 }, { "clip_ratio": 0.0, "completion_length": 659.9167022705078, "epoch": 0.9373333333333334, "grad_norm": 0.47496387362480164, "kl": 0.8115234375, "learning_rate": 3.5745116334828634e-08, "loss": 0.1476, "reward": 0.2083333395421505, "reward_std": 0.43528566509485245, "rewards/accuracy_reward": 0.2083333395421505, "rewards/format_reward": 0.0, "step": 1406 }, { "clip_ratio": 0.0, "completion_length": 522.0833435058594, "epoch": 0.938, "grad_norm": 1.1888548135757446, "kl": 0.84033203125, "learning_rate": 3.499158249874812e-08, "loss": 0.136, "reward": 0.3750000149011612, "reward_std": 0.43528568744659424, "rewards/accuracy_reward": 0.3750000149011612, "rewards/format_reward": 0.0, "step": 1407 }, { "clip_ratio": 0.0, "completion_length": 695.9583435058594, "epoch": 0.9386666666666666, "grad_norm": 0.5501106381416321, "kl": 1.63134765625, "learning_rate": 3.4245982296138126e-08, "loss": 0.163, "reward": 0.2500000074505806, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 1408 }, { "clip_ratio": 0.0, "completion_length": 761.8750152587891, "epoch": 0.9393333333333334, "grad_norm": 2.316314935684204, "kl": 1.787841796875, "learning_rate": 3.350831976473545e-08, "loss": 0.1598, "reward": 0.0833333358168602, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 1409 }, { "clip_ratio": 0.0, "completion_length": 620.2083511352539, "epoch": 0.94, "grad_norm": 0.47850456833839417, "kl": 0.8876953125, "learning_rate": 3.277859889929147e-08, "loss": 0.0941, "reward": 0.1666666716337204, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 1410 }, { "clip_ratio": 0.0, "completion_length": 704.3750152587891, "epoch": 0.9406666666666667, "grad_norm": 1.1566722393035889, "kl": 1.206787109375, "learning_rate": 3.2056823651550105e-08, "loss": 0.064, "reward": 0.2500000037252903, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.0, "step": 1411 }, { "clip_ratio": 0.0, "completion_length": 508.8333435058594, "epoch": 0.9413333333333334, "grad_norm": 1.8367347717285156, "kl": 1.26885986328125, "learning_rate": 3.1342997930225883e-08, "loss": 0.0632, "reward": 0.25, "reward_std": 0.0, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "step": 1412 }, { "clip_ratio": 0.0, "completion_length": 507.8333435058594, "epoch": 0.942, "grad_norm": 1.1878567934036255, "kl": 1.06689453125, "learning_rate": 3.063712560098392e-08, "loss": 0.0648, "reward": 0.5416666716337204, "reward_std": 0.539891391992569, "rewards/accuracy_reward": 0.5416666716337204, "rewards/format_reward": 0.0, "step": 1413 }, { "clip_ratio": 0.0, "completion_length": 714.0833587646484, "epoch": 0.9426666666666667, "grad_norm": 0.4011344611644745, "kl": 0.2603759765625, "learning_rate": 2.993921048641829e-08, "loss": 0.0376, "reward": 0.3750000111758709, "reward_std": 0.3410547897219658, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 1414 }, { "clip_ratio": 0.0, "completion_length": 532.2500152587891, "epoch": 0.9433333333333334, "grad_norm": 0.44604095816612244, "kl": 1.621826171875, "learning_rate": 2.9249256366031042e-08, "loss": 0.162, "reward": 0.3333333432674408, "reward_std": 0.3872983753681183, "rewards/accuracy_reward": 0.3333333432674408, "rewards/format_reward": 0.0, "step": 1415 }, { "clip_ratio": 0.0, "completion_length": 579.3333435058594, "epoch": 0.944, "grad_norm": 0.9342230558395386, "kl": 1.2353515625, "learning_rate": 2.8567266976212704e-08, "loss": 0.1039, "reward": 0.3750000074505806, "reward_std": 0.4701542407274246, "rewards/accuracy_reward": 0.3750000074505806, "rewards/format_reward": 0.0, "step": 1416 }, { "clip_ratio": 0.0, "completion_length": 634.6666793823242, "epoch": 0.9446666666666667, "grad_norm": 0.3414718508720398, "kl": 0.9237060546875, "learning_rate": 2.7893246010220974e-08, "loss": 0.0033, "reward": 0.3333333395421505, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.3333333395421505, "rewards/format_reward": 0.0, "step": 1417 }, { "clip_ratio": 0.0, "completion_length": 451.04168701171875, "epoch": 0.9453333333333334, "grad_norm": 1.5544121265411377, "kl": 1.16827392578125, "learning_rate": 2.722719711816124e-08, "loss": 0.1195, "reward": 0.6250000149011612, "reward_std": 0.5242290198802948, "rewards/accuracy_reward": 0.6250000149011612, "rewards/format_reward": 0.0, "step": 1418 }, { "clip_ratio": 0.0, "completion_length": 771.6250152587891, "epoch": 0.946, "grad_norm": 2.8882713317871094, "kl": 2.541015625, "learning_rate": 2.6569123906967087e-08, "loss": 0.1318, "reward": 0.0833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 1419 }, { "clip_ratio": 0.0, "completion_length": 558.9166870117188, "epoch": 0.9466666666666667, "grad_norm": 2.7326407432556152, "kl": 0.8863525390625, "learning_rate": 2.5919029940380145e-08, "loss": 0.0849, "reward": 0.2916666753590107, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.2916666753590107, "rewards/format_reward": 0.0, "step": 1420 }, { "clip_ratio": 0.0, "completion_length": 535.7500152587891, "epoch": 0.9473333333333334, "grad_norm": 0.6223140954971313, "kl": 0.987060546875, "learning_rate": 2.5276918738931286e-08, "loss": 0.1306, "reward": 0.1250000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 1421 }, { "clip_ratio": 0.0, "completion_length": 593.8750152587891, "epoch": 0.948, "grad_norm": 1.2022520303726196, "kl": 0.42633056640625, "learning_rate": 2.464279377992129e-08, "loss": 0.0785, "reward": 0.4583333507180214, "reward_std": 0.4893604442477226, "rewards/accuracy_reward": 0.4583333507180214, "rewards/format_reward": 0.0, "step": 1422 }, { "clip_ratio": 0.0, "completion_length": 582.2500152587891, "epoch": 0.9486666666666667, "grad_norm": 0.8775218725204468, "kl": 2.08062744140625, "learning_rate": 2.4016658497401867e-08, "loss": 0.2652, "reward": 0.5000000037252903, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.5000000037252903, "rewards/format_reward": 0.0, "step": 1423 }, { "clip_ratio": 0.0, "completion_length": 445.6666793823242, "epoch": 0.9493333333333334, "grad_norm": 1.6994928121566772, "kl": 0.35540771484375, "learning_rate": 2.3398516282158002e-08, "loss": 0.1069, "reward": 0.6250000149011612, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.6250000149011612, "rewards/format_reward": 0.0, "step": 1424 }, { "clip_ratio": 0.0, "completion_length": 417.0416717529297, "epoch": 0.95, "grad_norm": 1.053975224494934, "kl": 0.95068359375, "learning_rate": 2.278837048168797e-08, "loss": 0.1273, "reward": 0.3750000149011612, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.3750000149011612, "rewards/format_reward": 0.0, "step": 1425 }, { "clip_ratio": 0.0, "completion_length": 347.00001525878906, "epoch": 0.9506666666666667, "grad_norm": 0.2596506178379059, "kl": 0.298095703125, "learning_rate": 2.218622440018653e-08, "loss": 0.0127, "reward": 0.125, "reward_std": 0.1369306445121765, "rewards/accuracy_reward": 0.125, "rewards/format_reward": 0.0, "step": 1426 }, { "clip_ratio": 0.0, "completion_length": 470.9583435058594, "epoch": 0.9513333333333334, "grad_norm": 1.0632073879241943, "kl": 0.3154296875, "learning_rate": 2.1592081298527076e-08, "loss": 0.123, "reward": 0.4583333544433117, "reward_std": 0.3061862140893936, "rewards/accuracy_reward": 0.4583333544433117, "rewards/format_reward": 0.0, "step": 1427 }, { "clip_ratio": 0.0, "completion_length": 500.04168701171875, "epoch": 0.952, "grad_norm": 2.287665605545044, "kl": 1.6953125, "learning_rate": 2.1005944394242692e-08, "loss": 0.2864, "reward": 0.5416666828095913, "reward_std": 0.4701542481780052, "rewards/accuracy_reward": 0.5416666828095913, "rewards/format_reward": 0.0, "step": 1428 }, { "clip_ratio": 0.0, "completion_length": 274.29167556762695, "epoch": 0.9526666666666667, "grad_norm": 1.6931278705596924, "kl": 0.55908203125, "learning_rate": 2.042781686150996e-08, "loss": 0.016, "reward": 0.4583333544433117, "reward_std": 0.306186206638813, "rewards/accuracy_reward": 0.4583333544433117, "rewards/format_reward": 0.0, "step": 1429 }, { "clip_ratio": 0.0, "completion_length": 463.3333511352539, "epoch": 0.9533333333333334, "grad_norm": 0.9956976175308228, "kl": 0.373779296875, "learning_rate": 1.985770183113117e-08, "loss": 0.0071, "reward": 0.3750000111758709, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 1430 }, { "clip_ratio": 0.0, "completion_length": 664.0000152587891, "epoch": 0.954, "grad_norm": 0.7912677526473999, "kl": 1.085205078125, "learning_rate": 1.929560239051814e-08, "loss": 0.04, "reward": 0.0833333358168602, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 1431 }, { "clip_ratio": 0.0, "completion_length": 446.2916946411133, "epoch": 0.9546666666666667, "grad_norm": 0.8098444938659668, "kl": 1.003173828125, "learning_rate": 1.874152158367426e-08, "loss": 0.0701, "reward": 0.2083333358168602, "reward_std": 0.3410547897219658, "rewards/accuracy_reward": 0.2083333358168602, "rewards/format_reward": 0.0, "step": 1432 }, { "clip_ratio": 0.0, "completion_length": 574.8333587646484, "epoch": 0.9553333333333334, "grad_norm": 0.43212684988975525, "kl": 0.99169921875, "learning_rate": 1.819546241117881e-08, "loss": 0.067, "reward": 0.3750000111758709, "reward_std": 0.3602609932422638, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 1433 }, { "clip_ratio": 0.0, "completion_length": 491.1250114440918, "epoch": 0.956, "grad_norm": 0.6986437439918518, "kl": 1.03936767578125, "learning_rate": 1.7657427830170825e-08, "loss": 0.1162, "reward": 0.5416666716337204, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.5416666716337204, "rewards/format_reward": 0.0, "step": 1434 }, { "clip_ratio": 0.0, "completion_length": 376.75000381469727, "epoch": 0.9566666666666667, "grad_norm": 0.9094318747520447, "kl": 0.643798828125, "learning_rate": 1.7127420754332603e-08, "loss": 0.1128, "reward": 0.5000000074505806, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.5000000074505806, "rewards/format_reward": 0.0, "step": 1435 }, { "clip_ratio": 0.0, "completion_length": 339.83333587646484, "epoch": 0.9573333333333334, "grad_norm": 1.119210124015808, "kl": 0.7276611328125, "learning_rate": 1.6605444053874043e-08, "loss": 0.1207, "reward": 0.4583333432674408, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.4583333432674408, "rewards/format_reward": 0.0, "step": 1436 }, { "clip_ratio": 0.0, "completion_length": 622.5416870117188, "epoch": 0.958, "grad_norm": 1.0642212629318237, "kl": 0.50628662109375, "learning_rate": 1.60915005555175e-08, "loss": 0.0223, "reward": 0.1666666716337204, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 1437 }, { "clip_ratio": 0.0, "completion_length": 486.66668701171875, "epoch": 0.9586666666666667, "grad_norm": 1.5601868629455566, "kl": 1.04193115234375, "learning_rate": 1.5585593042482137e-08, "loss": 0.0674, "reward": 0.7083333507180214, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.7083333507180214, "rewards/format_reward": 0.0, "step": 1438 }, { "clip_ratio": 0.0, "completion_length": 597.1666793823242, "epoch": 0.9593333333333334, "grad_norm": 0.9072431325912476, "kl": 0.8212890625, "learning_rate": 1.5087724254468415e-08, "loss": 0.1477, "reward": 0.2916666679084301, "reward_std": 0.37592335790395737, "rewards/accuracy_reward": 0.2916666679084301, "rewards/format_reward": 0.0, "step": 1439 }, { "clip_ratio": 0.0, "completion_length": 522.9583435058594, "epoch": 0.96, "grad_norm": 3.0473132133483887, "kl": 1.111328125, "learning_rate": 1.4597896887644457e-08, "loss": 0.2449, "reward": 0.5000000223517418, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.5000000223517418, "rewards/format_reward": 0.0, "step": 1440 }, { "clip_ratio": 0.0, "completion_length": 626.0833435058594, "epoch": 0.9606666666666667, "grad_norm": 0.5850287675857544, "kl": 0.44012451171875, "learning_rate": 1.4116113594630387e-08, "loss": 0.0358, "reward": 0.2916666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.2916666679084301, "rewards/format_reward": 0.0, "step": 1441 }, { "clip_ratio": 0.0, "completion_length": 609.4166870117188, "epoch": 0.9613333333333334, "grad_norm": 1.0262123346328735, "kl": 1.65478515625, "learning_rate": 1.3642376984484006e-08, "loss": 0.2448, "reward": 0.5000000074505806, "reward_std": 0.3332236036658287, "rewards/accuracy_reward": 0.5000000074505806, "rewards/format_reward": 0.0, "step": 1442 }, { "clip_ratio": 0.0, "completion_length": 620.9166870117188, "epoch": 0.962, "grad_norm": 0.5242087244987488, "kl": 1.8857421875, "learning_rate": 1.3176689622687476e-08, "loss": 0.1908, "reward": 0.3333333469927311, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.3333333469927311, "rewards/format_reward": 0.0, "step": 1443 }, { "clip_ratio": 0.0, "completion_length": 560.2083549499512, "epoch": 0.9626666666666667, "grad_norm": 1.7917050123214722, "kl": 1.29119873046875, "learning_rate": 1.2719054031132493e-08, "loss": 0.0552, "reward": 0.1666666716337204, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 1444 }, { "clip_ratio": 0.0, "completion_length": 634.1250152587891, "epoch": 0.9633333333333334, "grad_norm": 0.4630538523197174, "kl": 1.1328125, "learning_rate": 1.2269472688107463e-08, "loss": 0.0721, "reward": 0.3333333395421505, "reward_std": 0.4971916303038597, "rewards/accuracy_reward": 0.3333333395421505, "rewards/format_reward": 0.0, "step": 1445 }, { "clip_ratio": 0.0, "completion_length": 463.7916793823242, "epoch": 0.964, "grad_norm": 0.6094815731048584, "kl": 0.65740966796875, "learning_rate": 1.1827948028283353e-08, "loss": 0.1048, "reward": 0.541666679084301, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.541666679084301, "rewards/format_reward": 0.0, "step": 1446 }, { "clip_ratio": 0.0, "completion_length": 787.2500152587891, "epoch": 0.9646666666666667, "grad_norm": 5.083098411560059, "kl": 1.3203125, "learning_rate": 1.1394482442700694e-08, "loss": 0.1129, "reward": 0.1250000037252903, "reward_std": 0.3061862140893936, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 1447 }, { "clip_ratio": 0.0, "completion_length": 586.2916793823242, "epoch": 0.9653333333333334, "grad_norm": 1.479184627532959, "kl": 1.2021484375, "learning_rate": 1.0969078278757261e-08, "loss": 0.1746, "reward": 0.3750000111758709, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.3750000111758709, "rewards/format_reward": 0.0, "step": 1448 }, { "clip_ratio": 0.0, "completion_length": 403.7916717529297, "epoch": 0.966, "grad_norm": 0.1507226824760437, "kl": 0.37725830078125, "learning_rate": 1.0551737840194587e-08, "loss": 0.0592, "reward": 0.5416666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.5416666679084301, "rewards/format_reward": 0.0, "step": 1449 }, { "clip_ratio": 0.0, "completion_length": 456.16668701171875, "epoch": 0.9666666666666667, "grad_norm": 0.7690902948379517, "kl": 0.4189453125, "learning_rate": 1.0142463387085465e-08, "loss": 0.1049, "reward": 0.416666679084301, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.416666679084301, "rewards/format_reward": 0.0, "step": 1450 }, { "clip_ratio": 0.0, "completion_length": 569.7083587646484, "epoch": 0.9673333333333334, "grad_norm": 1.0582383871078491, "kl": 1.95263671875, "learning_rate": 9.741257135822635e-09, "loss": 0.1501, "reward": 0.2916666716337204, "reward_std": 0.4701542481780052, "rewards/accuracy_reward": 0.2916666716337204, "rewards/format_reward": 0.0, "step": 1451 }, { "clip_ratio": 0.0, "completion_length": 554.7916793823242, "epoch": 0.968, "grad_norm": 0.43910735845565796, "kl": 0.731689453125, "learning_rate": 9.348121259105447e-09, "loss": 0.098, "reward": 0.5000000111758709, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.5000000111758709, "rewards/format_reward": 0.0, "step": 1452 }, { "clip_ratio": 0.0, "completion_length": 666.4166717529297, "epoch": 0.9686666666666667, "grad_norm": 0.4255465865135193, "kl": 0.698211669921875, "learning_rate": 8.963057885929382e-09, "loss": 0.0157, "reward": 0.2500000037252903, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.2500000037252903, "rewards/format_reward": 0.0, "step": 1453 }, { "clip_ratio": 0.0, "completion_length": 448.0416831970215, "epoch": 0.9693333333333334, "grad_norm": 1.4073446989059448, "kl": 1.013916015625, "learning_rate": 8.586069101573558e-09, "loss": 0.1226, "reward": 0.4583333469927311, "reward_std": 0.4893604442477226, "rewards/accuracy_reward": 0.4583333469927311, "rewards/format_reward": 0.0, "step": 1454 }, { "clip_ratio": 0.0, "completion_length": 558.1666946411133, "epoch": 0.97, "grad_norm": 1.1852753162384033, "kl": 0.87371826171875, "learning_rate": 8.217156947590065e-09, "loss": 0.0875, "reward": 0.375, "reward_std": 0.41079193353652954, "rewards/accuracy_reward": 0.375, "rewards/format_reward": 0.0, "step": 1455 }, { "clip_ratio": 0.0, "completion_length": 656.6666870117188, "epoch": 0.9706666666666667, "grad_norm": 2.501573085784912, "kl": 2.0703125, "learning_rate": 7.856323421792488e-09, "loss": 0.1774, "reward": 0.291666679084301, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.291666679084301, "rewards/format_reward": 0.0, "step": 1456 }, { "clip_ratio": 0.0, "completion_length": 519.5000152587891, "epoch": 0.9713333333333334, "grad_norm": 1.366695523262024, "kl": 0.94580078125, "learning_rate": 7.503570478245237e-09, "loss": 0.0763, "reward": 0.4583333507180214, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.4583333507180214, "rewards/format_reward": 0.0, "step": 1457 }, { "clip_ratio": 0.0, "completion_length": 346.8750114440918, "epoch": 0.972, "grad_norm": 2.6541664600372314, "kl": 1.47265625, "learning_rate": 7.158900027253223e-09, "loss": 0.1416, "reward": 0.4166666753590107, "reward_std": 0.4971916377544403, "rewards/accuracy_reward": 0.4166666753590107, "rewards/format_reward": 0.0, "step": 1458 }, { "clip_ratio": 0.0, "completion_length": 648.4583587646484, "epoch": 0.9726666666666667, "grad_norm": 0.5092136263847351, "kl": 0.99151611328125, "learning_rate": 6.82231393535121e-09, "loss": 0.0076, "reward": 0.3333333358168602, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 1459 }, { "clip_ratio": 0.0, "completion_length": 415.3750305175781, "epoch": 0.9733333333333334, "grad_norm": 0.4952450692653656, "kl": 0.5523681640625, "learning_rate": 6.493814025293476e-09, "loss": 0.0073, "reward": 0.4583333395421505, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.4583333395421505, "rewards/format_reward": 0.0, "step": 1460 }, { "clip_ratio": 0.0, "completion_length": 480.33334732055664, "epoch": 0.974, "grad_norm": 1.0471264123916626, "kl": 0.4913330078125, "learning_rate": 6.173402076044499e-09, "loss": 0.0902, "reward": 0.3333333358168602, "reward_std": 0.20412413775920868, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 1461 }, { "clip_ratio": 0.0, "completion_length": 690.5000152587891, "epoch": 0.9746666666666667, "grad_norm": 1.350835919380188, "kl": 1.02459716796875, "learning_rate": 5.861079822769122e-09, "loss": -0.043, "reward": 0.2500000111758709, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.2500000111758709, "rewards/format_reward": 0.0, "step": 1462 }, { "clip_ratio": 0.0, "completion_length": 362.50000762939453, "epoch": 0.9753333333333334, "grad_norm": 1.849319338798523, "kl": 0.673583984375, "learning_rate": 5.556848956823068e-09, "loss": 0.1341, "reward": 0.6250000149011612, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.6250000149011612, "rewards/format_reward": 0.0, "step": 1463 }, { "clip_ratio": 0.0, "completion_length": 611.2500228881836, "epoch": 0.976, "grad_norm": 0.4518433213233948, "kl": 0.51776123046875, "learning_rate": 5.260711125743445e-09, "loss": 0.0778, "reward": 0.3333333469927311, "reward_std": 0.40824829041957855, "rewards/accuracy_reward": 0.3333333469927311, "rewards/format_reward": 0.0, "step": 1464 }, { "clip_ratio": 0.0, "completion_length": 570.1250228881836, "epoch": 0.9766666666666667, "grad_norm": 0.4517037570476532, "kl": 0.4205322265625, "learning_rate": 4.972667933240916e-09, "loss": 0.0177, "reward": 0.0833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 1465 }, { "clip_ratio": 0.0, "completion_length": 536.5416793823242, "epoch": 0.9773333333333334, "grad_norm": 0.7896491885185242, "kl": 0.3983154296875, "learning_rate": 4.6927209391897095e-09, "loss": 0.1175, "reward": 0.4166666679084301, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.4166666679084301, "rewards/format_reward": 0.0, "step": 1466 }, { "clip_ratio": 0.0, "completion_length": 441.1666793823242, "epoch": 0.978, "grad_norm": 3.7440075874328613, "kl": 2.1624755859375, "learning_rate": 4.42087165961963e-09, "loss": 0.0696, "reward": 0.5416666716337204, "reward_std": 0.26603010296821594, "rewards/accuracy_reward": 0.5416666716337204, "rewards/format_reward": 0.0, "step": 1467 }, { "clip_ratio": 0.0, "completion_length": 593.2500228881836, "epoch": 0.9786666666666667, "grad_norm": 0.9797553420066833, "kl": 1.5234375, "learning_rate": 4.157121566708721e-09, "loss": 0.2006, "reward": 0.1666666679084301, "reward_std": 0.23899271339178085, "rewards/accuracy_reward": 0.1666666679084301, "rewards/format_reward": 0.0, "step": 1468 }, { "clip_ratio": 0.0, "completion_length": 671.0000305175781, "epoch": 0.9793333333333333, "grad_norm": 1.8547548055648804, "kl": 2.3046875, "learning_rate": 3.901472088773617e-09, "loss": 0.2023, "reward": 0.0833333358168602, "reward_std": 0.20412415266036987, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 1469 }, { "clip_ratio": 0.0, "completion_length": 400.9583435058594, "epoch": 0.98, "grad_norm": 0.39561209082603455, "kl": 0.975830078125, "learning_rate": 3.6539246102637037e-09, "loss": 0.1031, "reward": 0.5833333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.5833333358168602, "rewards/format_reward": 0.0, "step": 1470 }, { "clip_ratio": 0.0, "completion_length": 567.1250152587891, "epoch": 0.9806666666666667, "grad_norm": 0.6704661250114441, "kl": 0.459228515625, "learning_rate": 3.4144804717516354e-09, "loss": 0.0922, "reward": 0.2916666716337204, "reward_std": 0.39512956142425537, "rewards/accuracy_reward": 0.2916666716337204, "rewards/format_reward": 0.0, "step": 1471 }, { "clip_ratio": 0.0, "completion_length": 565.5000228881836, "epoch": 0.9813333333333333, "grad_norm": 1.6607613563537598, "kl": 2.251953125, "learning_rate": 3.183140969928e-09, "loss": 0.4084, "reward": 0.416666679084301, "reward_std": 0.3872983753681183, "rewards/accuracy_reward": 0.416666679084301, "rewards/format_reward": 0.0, "step": 1472 }, { "clip_ratio": 0.0, "completion_length": 740.0000305175781, "epoch": 0.982, "grad_norm": 1.053068995475769, "kl": 0.7694091796875, "learning_rate": 2.9599073575926614e-09, "loss": 0.0476, "reward": 0.4583333469927311, "reward_std": 0.4701542407274246, "rewards/accuracy_reward": 0.4583333469927311, "rewards/format_reward": 0.0, "step": 1473 }, { "clip_ratio": 0.0, "completion_length": 506.50001525878906, "epoch": 0.9826666666666667, "grad_norm": 0.7234020233154297, "kl": 1.90625, "learning_rate": 2.7447808436492637e-09, "loss": 0.0949, "reward": 0.5000000149011612, "reward_std": 0.3680921718478203, "rewards/accuracy_reward": 0.5000000149011612, "rewards/format_reward": 0.0, "step": 1474 }, { "clip_ratio": 0.0, "completion_length": 655.8333587646484, "epoch": 0.9833333333333333, "grad_norm": 0.4148600697517395, "kl": 0.8701171875, "learning_rate": 2.5377625930977367e-09, "loss": 0.0975, "reward": 0.1250000037252903, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 1475 }, { "clip_ratio": 0.0, "completion_length": 491.75001525878906, "epoch": 0.984, "grad_norm": 1.7435693740844727, "kl": 1.1197509765625, "learning_rate": 2.3388537270284673e-09, "loss": 0.0499, "reward": 0.6666666939854622, "reward_std": 0.46232306212186813, "rewards/accuracy_reward": 0.6666666939854622, "rewards/format_reward": 0.0, "step": 1476 }, { "clip_ratio": 0.0, "completion_length": 560.2916793823242, "epoch": 0.9846666666666667, "grad_norm": 0.9667103886604309, "kl": 1.056640625, "learning_rate": 2.148055322616471e-09, "loss": 0.1267, "reward": 0.0833333358168602, "reward_std": 0.20412414520978928, "rewards/accuracy_reward": 0.0833333358168602, "rewards/format_reward": 0.0, "step": 1477 }, { "clip_ratio": 0.0, "completion_length": 617.7083587646484, "epoch": 0.9853333333333333, "grad_norm": 2.5637707710266113, "kl": 1.658203125, "learning_rate": 1.9653684131145653e-09, "loss": 0.3014, "reward": 0.5416666828095913, "reward_std": 0.4701542481780052, "rewards/accuracy_reward": 0.5416666828095913, "rewards/format_reward": 0.0, "step": 1478 }, { "clip_ratio": 0.0, "completion_length": 446.3750114440918, "epoch": 0.986, "grad_norm": 1.7208720445632935, "kl": 1.30517578125, "learning_rate": 1.7907939878490376e-09, "loss": 0.0557, "reward": 0.5416666865348816, "reward_std": 0.3410547897219658, "rewards/accuracy_reward": 0.5416666865348816, "rewards/format_reward": 0.0, "step": 1479 }, { "clip_ratio": 0.0, "completion_length": 485.5833435058594, "epoch": 0.9866666666666667, "grad_norm": 0.43849512934684753, "kl": 0.5791015625, "learning_rate": 1.624332992213151e-09, "loss": -0.0044, "reward": 0.2916666679084301, "reward_std": 0.10206206887960434, "rewards/accuracy_reward": 0.2916666679084301, "rewards/format_reward": 0.0, "step": 1480 }, { "clip_ratio": 0.0, "completion_length": 620.6666870117188, "epoch": 0.9873333333333333, "grad_norm": 1.0228502750396729, "kl": 0.94317626953125, "learning_rate": 1.4659863276633157e-09, "loss": 0.1213, "reward": 0.2916666753590107, "reward_std": 0.3602609857916832, "rewards/accuracy_reward": 0.2916666753590107, "rewards/format_reward": 0.0, "step": 1481 }, { "clip_ratio": 0.0, "completion_length": 577.9583587646484, "epoch": 0.988, "grad_norm": 1.6231000423431396, "kl": 1.154296875, "learning_rate": 1.3157548517124252e-09, "loss": 0.2406, "reward": 0.3750000149011612, "reward_std": 0.3602609932422638, "rewards/accuracy_reward": 0.3750000149011612, "rewards/format_reward": 0.0, "step": 1482 }, { "clip_ratio": 0.0, "completion_length": 564.7500076293945, "epoch": 0.9886666666666667, "grad_norm": 0.630595862865448, "kl": 0.5736083984375, "learning_rate": 1.1736393779271938e-09, "loss": 0.0702, "reward": 0.6250000149011612, "reward_std": 0.23116153478622437, "rewards/accuracy_reward": 0.6250000149011612, "rewards/format_reward": 0.0, "step": 1483 }, { "clip_ratio": 0.0, "completion_length": 807.25, "epoch": 0.9893333333333333, "grad_norm": 3.3777973651885986, "kl": 1.7705078125, "learning_rate": 1.03964067592216e-09, "loss": 0.0741, "reward": 0.0, "reward_std": 0.0, "rewards/accuracy_reward": 0.0, "rewards/format_reward": 0.0, "step": 1484 }, { "clip_ratio": 0.0, "completion_length": 417.54168701171875, "epoch": 0.99, "grad_norm": 0.9785457849502563, "kl": 0.88922119140625, "learning_rate": 9.137594713563569e-10, "loss": 0.0567, "reward": 0.3333333358168602, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.3333333358168602, "rewards/format_reward": 0.0, "step": 1485 }, { "clip_ratio": 0.0, "completion_length": 510.2500228881836, "epoch": 0.9906666666666667, "grad_norm": 0.6635727882385254, "kl": 0.5137939453125, "learning_rate": 7.959964459293145e-10, "loss": 0.026, "reward": 0.1666666716337204, "reward_std": 0.12909945845603943, "rewards/accuracy_reward": 0.1666666716337204, "rewards/format_reward": 0.0, "step": 1486 }, { "clip_ratio": 0.0, "completion_length": 444.37501525878906, "epoch": 0.9913333333333333, "grad_norm": 0.5745776295661926, "kl": 1.14923095703125, "learning_rate": 6.863522373768971e-10, "loss": 0.1572, "reward": 0.7083333395421505, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.7083333395421505, "rewards/format_reward": 0.0, "step": 1487 }, { "clip_ratio": 0.0, "completion_length": 600.8750152587891, "epoch": 0.992, "grad_norm": 1.1147081851959229, "kl": 1.20263671875, "learning_rate": 5.848274394684716e-10, "loss": 0.1484, "reward": 0.2916666716337204, "reward_std": 0.39512956142425537, "rewards/accuracy_reward": 0.2916666716337204, "rewards/format_reward": 0.0, "step": 1488 }, { "clip_ratio": 0.0, "completion_length": 442.20834732055664, "epoch": 0.9926666666666667, "grad_norm": 0.38045838475227356, "kl": 1.25634765625, "learning_rate": 4.914226020032441e-10, "loss": 0.1394, "reward": 0.4583333469927311, "reward_std": 0.4701542556285858, "rewards/accuracy_reward": 0.4583333469927311, "rewards/format_reward": 0.0, "step": 1489 }, { "clip_ratio": 0.0, "completion_length": 551.4583511352539, "epoch": 0.9933333333333333, "grad_norm": 0.4784080982208252, "kl": 1.11419677734375, "learning_rate": 4.0613823080742907e-10, "loss": 0.159, "reward": 0.3750000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.3750000037252903, "rewards/format_reward": 0.0, "step": 1490 }, { "clip_ratio": 0.0, "completion_length": 509.4166717529297, "epoch": 0.994, "grad_norm": 0.8975785374641418, "kl": 0.65234375, "learning_rate": 3.289747877317506e-10, "loss": -0.0225, "reward": 0.375, "reward_std": 0.1369306445121765, "rewards/accuracy_reward": 0.375, "rewards/format_reward": 0.0, "step": 1491 }, { "clip_ratio": 0.0, "completion_length": 676.3750305175781, "epoch": 0.9946666666666667, "grad_norm": 3.1245534420013428, "kl": 1.176422119140625, "learning_rate": 2.599326906484456e-10, "loss": 0.0833, "reward": 0.4583333358168602, "reward_std": 0.3410547822713852, "rewards/accuracy_reward": 0.4583333358168602, "rewards/format_reward": 0.0, "step": 1492 }, { "clip_ratio": 0.0, "completion_length": 646.4583587646484, "epoch": 0.9953333333333333, "grad_norm": 0.946910560131073, "kl": 1.31298828125, "learning_rate": 1.9901231344926496e-10, "loss": 0.0517, "reward": 0.0416666679084301, "reward_std": 0.10206207633018494, "rewards/accuracy_reward": 0.0416666679084301, "rewards/format_reward": 0.0, "step": 1493 }, { "clip_ratio": 0.0, "completion_length": 670.25, "epoch": 0.996, "grad_norm": 1.5324867963790894, "kl": 0.929931640625, "learning_rate": 1.4621398604364178e-10, "loss": 0.083, "reward": 0.3750000149011612, "reward_std": 0.43528568744659424, "rewards/accuracy_reward": 0.3750000149011612, "rewards/format_reward": 0.0, "step": 1494 }, { "clip_ratio": 0.0, "completion_length": 353.2500228881836, "epoch": 0.9966666666666667, "grad_norm": 0.7476786971092224, "kl": 0.6732177734375, "learning_rate": 1.0153799435669297e-10, "loss": 0.0734, "reward": 0.5416666716337204, "reward_std": 0.39512956142425537, "rewards/accuracy_reward": 0.5416666716337204, "rewards/format_reward": 0.0, "step": 1495 }, { "clip_ratio": 0.0, "completion_length": 646.7917022705078, "epoch": 0.9973333333333333, "grad_norm": 0.8671073913574219, "kl": 1.5625, "learning_rate": 6.498458032738741e-11, "loss": 0.2392, "reward": 0.2500000074505806, "reward_std": 0.25819891691207886, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 1496 }, { "clip_ratio": 0.0, "completion_length": 314.2916793823242, "epoch": 0.998, "grad_norm": 0.5206292867660522, "kl": 0.298492431640625, "learning_rate": 3.655394190787975e-11, "loss": 0.0172, "reward": 0.25, "reward_std": 0.0, "rewards/accuracy_reward": 0.25, "rewards/format_reward": 0.0, "step": 1497 }, { "clip_ratio": 0.0, "completion_length": 565.5833435058594, "epoch": 0.9986666666666667, "grad_norm": 0.45190152525901794, "kl": 0.45025634765625, "learning_rate": 1.624623306184514e-11, "loss": 0.0388, "reward": 0.1250000037252903, "reward_std": 0.23116152733564377, "rewards/accuracy_reward": 0.1250000037252903, "rewards/format_reward": 0.0, "step": 1498 }, { "clip_ratio": 0.0, "completion_length": 407.9583435058594, "epoch": 0.9993333333333333, "grad_norm": 0.4099542200565338, "kl": 0.5985107421875, "learning_rate": 4.061563764312659e-12, "loss": 0.0489, "reward": 0.2500000074505806, "reward_std": 0.3872983753681183, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 1499 }, { "clip_ratio": 0.0, "completion_length": 577.9166717529297, "epoch": 1.0, "grad_norm": 0.8833712339401245, "kl": 0.953125, "learning_rate": 0.0, "loss": 0.0734, "reward": 0.2500000074505806, "reward_std": 0.3332235962152481, "rewards/accuracy_reward": 0.2500000074505806, "rewards/format_reward": 0.0, "step": 1500 }, { "epoch": 1.0, "step": 1500, "total_flos": 0.0, "train_loss": 0.09189987016815242, "train_runtime": 86246.2602, "train_samples_per_second": 0.07, "train_steps_per_second": 0.017 } ], "logging_steps": 1, "max_steps": 1500, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 500, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 2, "trial_name": null, "trial_params": null }