| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.9952153110047846, | |
| "eval_steps": 500, | |
| "global_step": 104, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 885.7360992431641, | |
| "epoch": 0.019138755980861243, | |
| "grad_norm": 0.0571745065537259, | |
| "kl": 0.0, | |
| "learning_rate": 9.09090909090909e-08, | |
| "loss": 0.0224, | |
| "reward": 0.6623263955116272, | |
| "reward_std": 0.20746709778904915, | |
| "rewards/accuracy_reward": 0.2500000037252903, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4123263880610466, | |
| "step": 1 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 855.6076354980469, | |
| "epoch": 0.03827751196172249, | |
| "grad_norm": 0.06704915151383242, | |
| "kl": 0.0, | |
| "learning_rate": 1.818181818181818e-07, | |
| "loss": 0.0199, | |
| "reward": 0.6901041716337204, | |
| "reward_std": 0.22956417128443718, | |
| "rewards/accuracy_reward": 0.2465277761220932, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4435763880610466, | |
| "step": 2 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 904.9861297607422, | |
| "epoch": 0.05741626794258373, | |
| "grad_norm": 0.05238384111260958, | |
| "kl": 2.7120113372802734e-05, | |
| "learning_rate": 2.727272727272727e-07, | |
| "loss": 0.0109, | |
| "reward": 0.5494791641831398, | |
| "reward_std": 0.17554907500743866, | |
| "rewards/accuracy_reward": 0.17361111380159855, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.375868059694767, | |
| "step": 3 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 880.2083435058594, | |
| "epoch": 0.07655502392344497, | |
| "grad_norm": 0.05735410600314712, | |
| "kl": 2.5987625122070312e-05, | |
| "learning_rate": 3.636363636363636e-07, | |
| "loss": 0.0214, | |
| "reward": 0.6362847238779068, | |
| "reward_std": 0.22956416755914688, | |
| "rewards/accuracy_reward": 0.2256944440305233, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4105902835726738, | |
| "step": 4 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 867.8368225097656, | |
| "epoch": 0.09569377990430622, | |
| "grad_norm": 0.056581697743245854, | |
| "kl": 2.8789043426513672e-05, | |
| "learning_rate": 4.545454545454545e-07, | |
| "loss": 0.0117, | |
| "reward": 0.665798619389534, | |
| "reward_std": 0.1829147655516863, | |
| "rewards/accuracy_reward": 0.25, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4157986119389534, | |
| "step": 5 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 884.0381927490234, | |
| "epoch": 0.11483253588516747, | |
| "grad_norm": 0.060995509922658335, | |
| "kl": 2.4437904357910156e-05, | |
| "learning_rate": 5.454545454545454e-07, | |
| "loss": 0.0206, | |
| "reward": 0.659722238779068, | |
| "reward_std": 0.24306795001029968, | |
| "rewards/accuracy_reward": 0.243055559694767, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4166666716337204, | |
| "step": 6 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 878.7083435058594, | |
| "epoch": 0.1339712918660287, | |
| "grad_norm": 0.06312478683496295, | |
| "kl": 2.7120113372802734e-05, | |
| "learning_rate": 6.363636363636363e-07, | |
| "loss": 0.0152, | |
| "reward": 0.6484375149011612, | |
| "reward_std": 0.224653709679842, | |
| "rewards/accuracy_reward": 0.243055559694767, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4053819477558136, | |
| "step": 7 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 871.4930572509766, | |
| "epoch": 0.15311004784688995, | |
| "grad_norm": 0.06419727373029128, | |
| "kl": 2.6404857635498047e-05, | |
| "learning_rate": 7.272727272727272e-07, | |
| "loss": 0.0246, | |
| "reward": 0.6631944477558136, | |
| "reward_std": 0.2381574772298336, | |
| "rewards/accuracy_reward": 0.2395833283662796, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.423611119389534, | |
| "step": 8 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 883.0347290039062, | |
| "epoch": 0.1722488038277512, | |
| "grad_norm": 0.07232208605276093, | |
| "kl": 2.5197863578796387e-05, | |
| "learning_rate": 8.181818181818182e-07, | |
| "loss": 0.0217, | |
| "reward": 0.6892361044883728, | |
| "reward_std": 0.270075511187315, | |
| "rewards/accuracy_reward": 0.2847222164273262, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4045138880610466, | |
| "step": 9 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 869.8229217529297, | |
| "epoch": 0.19138755980861244, | |
| "grad_norm": 0.05684777755384174, | |
| "kl": 2.7745962142944336e-05, | |
| "learning_rate": 9.09090909090909e-07, | |
| "loss": 0.0053, | |
| "reward": 0.697048619389534, | |
| "reward_std": 0.2197432443499565, | |
| "rewards/accuracy_reward": 0.2916666641831398, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4053819477558136, | |
| "step": 10 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 850.7152862548828, | |
| "epoch": 0.21052631578947367, | |
| "grad_norm": 0.07180408830721081, | |
| "kl": 2.5004148483276367e-05, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0299, | |
| "reward": 0.706597238779068, | |
| "reward_std": 0.28726211935281754, | |
| "rewards/accuracy_reward": 0.2673611156642437, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.439236119389534, | |
| "step": 11 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 868.28125, | |
| "epoch": 0.22966507177033493, | |
| "grad_norm": 0.0646871027868035, | |
| "kl": 2.7686357498168945e-05, | |
| "learning_rate": 9.99743270918373e-07, | |
| "loss": 0.0177, | |
| "reward": 0.7300347238779068, | |
| "reward_std": 0.24429556727409363, | |
| "rewards/accuracy_reward": 0.2881944440305233, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4418402761220932, | |
| "step": 12 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 864.7395935058594, | |
| "epoch": 0.24880382775119617, | |
| "grad_norm": 0.06556460276965595, | |
| "kl": 2.7000904083251953e-05, | |
| "learning_rate": 9.989733766060318e-07, | |
| "loss": 0.0163, | |
| "reward": 0.712673619389534, | |
| "reward_std": 0.20255662500858307, | |
| "rewards/accuracy_reward": 0.2777777910232544, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4348958283662796, | |
| "step": 13 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 879.0590209960938, | |
| "epoch": 0.2679425837320574, | |
| "grad_norm": 0.06296015481337248, | |
| "kl": 3.0308961868286133e-05, | |
| "learning_rate": 9.976911955263528e-07, | |
| "loss": 0.0101, | |
| "reward": 0.6510416716337204, | |
| "reward_std": 0.22588133066892624, | |
| "rewards/accuracy_reward": 0.2326388917863369, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4184027761220932, | |
| "step": 14 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 885.1006927490234, | |
| "epoch": 0.28708133971291866, | |
| "grad_norm": 0.06103661524458471, | |
| "kl": 3.299117088317871e-05, | |
| "learning_rate": 9.958981906712076e-07, | |
| "loss": 0.0203, | |
| "reward": 0.677083358168602, | |
| "reward_std": 0.272530734539032, | |
| "rewards/accuracy_reward": 0.2534722238779068, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4236111119389534, | |
| "step": 15 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 874.2257080078125, | |
| "epoch": 0.3062200956937799, | |
| "grad_norm": 0.0764541954488641, | |
| "kl": 5.513429641723633e-05, | |
| "learning_rate": 9.935964078916617e-07, | |
| "loss": 0.0251, | |
| "reward": 0.681423619389534, | |
| "reward_std": 0.21483278647065163, | |
| "rewards/accuracy_reward": 0.2708333395421505, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4105902835726738, | |
| "step": 16 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 852.8472290039062, | |
| "epoch": 0.3253588516746411, | |
| "grad_norm": 0.0763344765825447, | |
| "kl": 0.00010287761688232422, | |
| "learning_rate": 9.907884735636224e-07, | |
| "loss": 0.0222, | |
| "reward": 0.7560763955116272, | |
| "reward_std": 0.2909449674189091, | |
| "rewards/accuracy_reward": 0.3263888880610466, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4296875074505806, | |
| "step": 17 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 873.6388854980469, | |
| "epoch": 0.3444976076555024, | |
| "grad_norm": 0.07015709920099024, | |
| "kl": 0.0001728534698486328, | |
| "learning_rate": 9.874775915910958e-07, | |
| "loss": 0.0186, | |
| "reward": 0.7039930671453476, | |
| "reward_std": 0.224653709679842, | |
| "rewards/accuracy_reward": 0.274305559694767, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4296875, | |
| "step": 18 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 865.6250152587891, | |
| "epoch": 0.36363636363636365, | |
| "grad_norm": 0.06286838529602348, | |
| "kl": 0.0002143383026123047, | |
| "learning_rate": 9.836675397504734e-07, | |
| "loss": 0.0188, | |
| "reward": 0.724826380610466, | |
| "reward_std": 0.23938511312007904, | |
| "rewards/accuracy_reward": 0.3090277835726738, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.415798619389534, | |
| "step": 19 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 851.5555572509766, | |
| "epoch": 0.3827751196172249, | |
| "grad_norm": 0.0652308138114061, | |
| "kl": 0.00026702880859375, | |
| "learning_rate": 9.793626653800218e-07, | |
| "loss": 0.019, | |
| "reward": 0.7230902761220932, | |
| "reward_std": 0.24184033274650574, | |
| "rewards/accuracy_reward": 0.288194440305233, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4348958283662796, | |
| "step": 20 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 861.40625, | |
| "epoch": 0.4019138755980861, | |
| "grad_norm": 0.054173134249339484, | |
| "kl": 0.00030875205993652344, | |
| "learning_rate": 9.74567880419491e-07, | |
| "loss": 0.0092, | |
| "reward": 0.6545138955116272, | |
| "reward_std": 0.2086947076022625, | |
| "rewards/accuracy_reward": 0.2361111119389534, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4184027835726738, | |
| "step": 21 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 839.1805725097656, | |
| "epoch": 0.42105263157894735, | |
| "grad_norm": 0.06484920729256737, | |
| "kl": 0.0003552436828613281, | |
| "learning_rate": 9.692886558055006e-07, | |
| "loss": 0.019, | |
| "reward": 0.7100694626569748, | |
| "reward_std": 0.23079179227352142, | |
| "rewards/accuracy_reward": 0.2881944440305233, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4218749925494194, | |
| "step": 22 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 889.3159790039062, | |
| "epoch": 0.44019138755980863, | |
| "grad_norm": 0.0673083530273057, | |
| "kl": 0.0004329681396484375, | |
| "learning_rate": 9.635310152291037e-07, | |
| "loss": 0.02, | |
| "reward": 0.6163194626569748, | |
| "reward_std": 0.20132901147007942, | |
| "rewards/accuracy_reward": 0.2222222201526165, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.3940972238779068, | |
| "step": 23 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 872.4722290039062, | |
| "epoch": 0.45933014354066987, | |
| "grad_norm": 0.05489657330998097, | |
| "kl": 0.0005602836608886719, | |
| "learning_rate": 9.573015282626445e-07, | |
| "loss": 0.0217, | |
| "reward": 0.6996527910232544, | |
| "reward_std": 0.22097085788846016, | |
| "rewards/accuracy_reward": 0.274305559694767, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4253472313284874, | |
| "step": 24 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 864.3889007568359, | |
| "epoch": 0.4784688995215311, | |
| "grad_norm": 0.06394072732120833, | |
| "kl": 0.0007648468017578125, | |
| "learning_rate": 9.506073028637582e-07, | |
| "loss": 0.0165, | |
| "reward": 0.6866319477558136, | |
| "reward_std": 0.26148219406604767, | |
| "rewards/accuracy_reward": 0.274305559694767, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4123263880610466, | |
| "step": 25 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 859.4375152587891, | |
| "epoch": 0.49760765550239233, | |
| "grad_norm": 0.06801792884391539, | |
| "kl": 0.0011510848999023438, | |
| "learning_rate": 9.434559772650618e-07, | |
| "loss": 0.0179, | |
| "reward": 0.7569444477558136, | |
| "reward_std": 0.2357022576034069, | |
| "rewards/accuracy_reward": 0.329861119389534, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4270833358168602, | |
| "step": 26 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 882.1701354980469, | |
| "epoch": 0.5167464114832536, | |
| "grad_norm": 0.06594394459261921, | |
| "kl": 0.001209259033203125, | |
| "learning_rate": 9.358557112587928e-07, | |
| "loss": 0.0201, | |
| "reward": 0.6857638955116272, | |
| "reward_std": 0.2651650384068489, | |
| "rewards/accuracy_reward": 0.2777777761220932, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.407986119389534, | |
| "step": 27 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 848.2395935058594, | |
| "epoch": 0.5358851674641149, | |
| "grad_norm": 0.0697274048868991, | |
| "kl": 0.001598358154296875, | |
| "learning_rate": 9.278151768863375e-07, | |
| "loss": 0.029, | |
| "reward": 0.7039930671453476, | |
| "reward_std": 0.2835792750120163, | |
| "rewards/accuracy_reward": 0.2881944440305233, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4157986119389534, | |
| "step": 28 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 858.8889007568359, | |
| "epoch": 0.5550239234449761, | |
| "grad_norm": 0.06562680518580778, | |
| "kl": 0.0018520355224609375, | |
| "learning_rate": 9.193435485432744e-07, | |
| "loss": 0.0227, | |
| "reward": 0.7395833283662796, | |
| "reward_std": 0.2676202692091465, | |
| "rewards/accuracy_reward": 0.3159722313284874, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.423611119389534, | |
| "step": 29 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 809.0694580078125, | |
| "epoch": 0.5741626794258373, | |
| "grad_norm": 0.07438927904369452, | |
| "kl": 0.002056121826171875, | |
| "learning_rate": 9.104504925112231e-07, | |
| "loss": 0.0256, | |
| "reward": 0.8541666716337204, | |
| "reward_std": 0.2749859616160393, | |
| "rewards/accuracy_reward": 0.399305559694767, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.454861119389534, | |
| "step": 30 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 871.6979217529297, | |
| "epoch": 0.5933014354066986, | |
| "grad_norm": 0.049564487099522436, | |
| "kl": 0.00186920166015625, | |
| "learning_rate": 9.011461559284413e-07, | |
| "loss": 0.0095, | |
| "reward": 0.7387152910232544, | |
| "reward_std": 0.15099675580859184, | |
| "rewards/accuracy_reward": 0.3402777835726738, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.3984375074505806, | |
| "step": 31 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 834.6250152587891, | |
| "epoch": 0.6124401913875598, | |
| "grad_norm": 0.06896614921110253, | |
| "kl": 0.0023975372314453125, | |
| "learning_rate": 8.914411552117559e-07, | |
| "loss": 0.0227, | |
| "reward": 0.7560763955116272, | |
| "reward_std": 0.26884788274765015, | |
| "rewards/accuracy_reward": 0.336805559694767, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4192708358168602, | |
| "step": 32 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 845.7048645019531, | |
| "epoch": 0.631578947368421, | |
| "grad_norm": 0.07262913185512221, | |
| "kl": 0.00289154052734375, | |
| "learning_rate": 8.813465639430386e-07, | |
| "loss": 0.0293, | |
| "reward": 0.7838541716337204, | |
| "reward_std": 0.26393742114305496, | |
| "rewards/accuracy_reward": 0.329861119389534, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4539930522441864, | |
| "step": 33 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 825.7361145019531, | |
| "epoch": 0.6507177033492823, | |
| "grad_norm": 0.06923687691149574, | |
| "kl": 0.002803802490234375, | |
| "learning_rate": 8.70873900234049e-07, | |
| "loss": 0.0156, | |
| "reward": 0.8272569626569748, | |
| "reward_std": 0.2590269446372986, | |
| "rewards/accuracy_reward": 0.392361119389534, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4348958358168602, | |
| "step": 34 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 813.9930725097656, | |
| "epoch": 0.6698564593301436, | |
| "grad_norm": 0.0634139401535218, | |
| "kl": 0.002788543701171875, | |
| "learning_rate": 8.600351135840589e-07, | |
| "loss": 0.0179, | |
| "reward": 0.8159722238779068, | |
| "reward_std": 0.24306795746088028, | |
| "rewards/accuracy_reward": 0.3715277835726738, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4444444477558136, | |
| "step": 35 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 888.8159942626953, | |
| "epoch": 0.6889952153110048, | |
| "grad_norm": 0.06115251503429904, | |
| "kl": 0.00292205810546875, | |
| "learning_rate": 8.488425712452576e-07, | |
| "loss": 0.0181, | |
| "reward": 0.6588541716337204, | |
| "reward_std": 0.2221984826028347, | |
| "rewards/accuracy_reward": 0.2777777798473835, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.3810763880610466, | |
| "step": 36 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 837.6458435058594, | |
| "epoch": 0.7081339712918661, | |
| "grad_norm": 0.06174697280825877, | |
| "kl": 0.002880096435546875, | |
| "learning_rate": 8.373090441114925e-07, | |
| "loss": 0.0255, | |
| "reward": 0.7873263955116272, | |
| "reward_std": 0.2197432555258274, | |
| "rewards/accuracy_reward": 0.3541666716337204, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4331597238779068, | |
| "step": 37 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 855.8090362548828, | |
| "epoch": 0.7272727272727273, | |
| "grad_norm": 0.07806365108296032, | |
| "kl": 0.00281524658203125, | |
| "learning_rate": 8.254476921464483e-07, | |
| "loss": 0.0277, | |
| "reward": 0.7621527910232544, | |
| "reward_std": 0.24797841161489487, | |
| "rewards/accuracy_reward": 0.3298611119389534, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4322916716337204, | |
| "step": 38 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 857.6840515136719, | |
| "epoch": 0.7464114832535885, | |
| "grad_norm": 0.07203148213149006, | |
| "kl": 0.003299713134765625, | |
| "learning_rate": 8.132720493678884e-07, | |
| "loss": 0.023, | |
| "reward": 0.7803819328546524, | |
| "reward_std": 0.24184033647179604, | |
| "rewards/accuracy_reward": 0.3402777835726738, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4401041716337204, | |
| "step": 39 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 838.0764007568359, | |
| "epoch": 0.7655502392344498, | |
| "grad_norm": 0.06474279173647825, | |
| "kl": 0.00308990478515625, | |
| "learning_rate": 8.007960084050963e-07, | |
| "loss": 0.0212, | |
| "reward": 0.8741319477558136, | |
| "reward_std": 0.23447464406490326, | |
| "rewards/accuracy_reward": 0.4340277761220932, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4401041716337204, | |
| "step": 40 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 859.78125, | |
| "epoch": 0.784688995215311, | |
| "grad_norm": 0.06299207628008677, | |
| "kl": 0.003017425537109375, | |
| "learning_rate": 7.88033804647133e-07, | |
| "loss": 0.0193, | |
| "reward": 0.7256944626569748, | |
| "reward_std": 0.24306795746088028, | |
| "rewards/accuracy_reward": 0.3159722238779068, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4097222238779068, | |
| "step": 41 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 840.0625152587891, | |
| "epoch": 0.8038277511961722, | |
| "grad_norm": 0.07344429258616171, | |
| "kl": 0.003047943115234375, | |
| "learning_rate": 7.75e-07, | |
| "loss": 0.0251, | |
| "reward": 0.8220486044883728, | |
| "reward_std": 0.20992231741547585, | |
| "rewards/accuracy_reward": 0.392361119389534, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4296875074505806, | |
| "step": 42 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 819.0382232666016, | |
| "epoch": 0.8229665071770335, | |
| "grad_norm": 0.061499395941431584, | |
| "kl": 0.003070831298828125, | |
| "learning_rate": 7.6170946627124e-07, | |
| "loss": 0.0223, | |
| "reward": 0.8151041716337204, | |
| "reward_std": 0.20992232859134674, | |
| "rewards/accuracy_reward": 0.3784722164273262, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4366319477558136, | |
| "step": 43 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 863.3264007568359, | |
| "epoch": 0.8421052631578947, | |
| "grad_norm": 0.06745163743583922, | |
| "kl": 0.0031585693359375, | |
| "learning_rate": 7.481773682009354e-07, | |
| "loss": 0.0196, | |
| "reward": 0.7387152910232544, | |
| "reward_std": 0.20501185581088066, | |
| "rewards/accuracy_reward": 0.3159722238779068, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.422743059694767, | |
| "step": 44 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 830.9930725097656, | |
| "epoch": 0.861244019138756, | |
| "grad_norm": 0.07628628253732472, | |
| "kl": 0.0036468505859375, | |
| "learning_rate": 7.344191461584643e-07, | |
| "loss": 0.0256, | |
| "reward": 0.8289930671453476, | |
| "reward_std": 0.2492060288786888, | |
| "rewards/accuracy_reward": 0.3819444477558136, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4470486119389534, | |
| "step": 45 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 834.5590362548828, | |
| "epoch": 0.8803827751196173, | |
| "grad_norm": 0.0697101918978914, | |
| "kl": 0.004314422607421875, | |
| "learning_rate": 7.204504985247592e-07, | |
| "loss": 0.0163, | |
| "reward": 0.8498263955116272, | |
| "reward_std": 0.24184034019708633, | |
| "rewards/accuracy_reward": 0.40625, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4435763955116272, | |
| "step": 46 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 846.2569427490234, | |
| "epoch": 0.8995215311004785, | |
| "grad_norm": 0.06859248959883628, | |
| "kl": 0.003566741943359375, | |
| "learning_rate": 7.06287363780169e-07, | |
| "loss": 0.0134, | |
| "reward": 0.7534722238779068, | |
| "reward_std": 0.22097086533904076, | |
| "rewards/accuracy_reward": 0.3333333358168602, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4201388955116272, | |
| "step": 47 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 813.9062652587891, | |
| "epoch": 0.9186602870813397, | |
| "grad_norm": 0.07616625495450065, | |
| "kl": 0.00397491455078125, | |
| "learning_rate": 6.91945902318365e-07, | |
| "loss": 0.0284, | |
| "reward": 0.862847238779068, | |
| "reward_std": 0.2700754962861538, | |
| "rewards/accuracy_reward": 0.4131944477558136, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4496527761220932, | |
| "step": 48 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 843.1215209960938, | |
| "epoch": 0.937799043062201, | |
| "grad_norm": 0.06987264134329038, | |
| "kl": 0.003589630126953125, | |
| "learning_rate": 6.774424780070367e-07, | |
| "loss": 0.0194, | |
| "reward": 0.8289930671453476, | |
| "reward_std": 0.2786688208580017, | |
| "rewards/accuracy_reward": 0.4027777761220932, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4262152835726738, | |
| "step": 49 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 848.1909790039062, | |
| "epoch": 0.9569377990430622, | |
| "grad_norm": 0.07535010112426463, | |
| "kl": 0.00441741943359375, | |
| "learning_rate": 6.627936395164242e-07, | |
| "loss": 0.0225, | |
| "reward": 0.8229166716337204, | |
| "reward_std": 0.27744119614362717, | |
| "rewards/accuracy_reward": 0.3645833358168602, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4583333358168602, | |
| "step": 50 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 844.8402862548828, | |
| "epoch": 0.9760765550239234, | |
| "grad_norm": 0.07005632661144727, | |
| "kl": 0.004360198974609375, | |
| "learning_rate": 6.480161014369832e-07, | |
| "loss": 0.0217, | |
| "reward": 0.8645833283662796, | |
| "reward_std": 0.25043364986777306, | |
| "rewards/accuracy_reward": 0.4270833358168602, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4375, | |
| "step": 51 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 859.1458587646484, | |
| "epoch": 0.9952153110047847, | |
| "grad_norm": 0.07899639612660056, | |
| "kl": 0.004245758056640625, | |
| "learning_rate": 6.331267252077352e-07, | |
| "loss": 0.0249, | |
| "reward": 0.8576388955116272, | |
| "reward_std": 0.25043364241719246, | |
| "rewards/accuracy_reward": 0.423611119389534, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4340277761220932, | |
| "step": 52 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 818.5764007568359, | |
| "epoch": 1.0191387559808613, | |
| "grad_norm": 0.06800368909135272, | |
| "kl": 0.0054168701171875, | |
| "learning_rate": 6.181424998770595e-07, | |
| "loss": 0.0099, | |
| "reward": 0.9149305522441864, | |
| "reward_std": 0.25534411147236824, | |
| "rewards/accuracy_reward": 0.444444440305233, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4704861044883728, | |
| "step": 53 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 825.7847290039062, | |
| "epoch": 1.0382775119617225, | |
| "grad_norm": 0.0687845544933541, | |
| "kl": 0.00484466552734375, | |
| "learning_rate": 6.030805227178808e-07, | |
| "loss": 0.0266, | |
| "reward": 0.8932291716337204, | |
| "reward_std": 0.27130311727523804, | |
| "rewards/accuracy_reward": 0.4236111119389534, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.469618059694767, | |
| "step": 54 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 831.6875152587891, | |
| "epoch": 1.0574162679425838, | |
| "grad_norm": 0.06231563710215865, | |
| "kl": 0.0048065185546875, | |
| "learning_rate": 5.879579797193701e-07, | |
| "loss": 0.0051, | |
| "reward": 0.8385416716337204, | |
| "reward_std": 0.23324701189994812, | |
| "rewards/accuracy_reward": 0.3958333358168602, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4427083358168602, | |
| "step": 55 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 813.4722290039062, | |
| "epoch": 1.076555023923445, | |
| "grad_norm": 0.07317349080061789, | |
| "kl": 0.0054473876953125, | |
| "learning_rate": 5.727921259774208e-07, | |
| "loss": 0.0342, | |
| "reward": 0.8819444626569748, | |
| "reward_std": 0.25043365359306335, | |
| "rewards/accuracy_reward": 0.4236111268401146, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4583333432674408, | |
| "step": 56 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 802.3784942626953, | |
| "epoch": 1.0956937799043063, | |
| "grad_norm": 0.07553172071799115, | |
| "kl": 0.00548553466796875, | |
| "learning_rate": 5.57600266006269e-07, | |
| "loss": 0.0246, | |
| "reward": 0.9357638955116272, | |
| "reward_std": 0.2970830574631691, | |
| "rewards/accuracy_reward": 0.4583333432674408, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4774305671453476, | |
| "step": 57 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 809.1076354980469, | |
| "epoch": 1.1148325358851674, | |
| "grad_norm": 0.08337759523447522, | |
| "kl": 0.0051727294921875, | |
| "learning_rate": 5.423997339937311e-07, | |
| "loss": 0.0237, | |
| "reward": 0.8854167014360428, | |
| "reward_std": 0.3019935190677643, | |
| "rewards/accuracy_reward": 0.4097222238779068, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4756944477558136, | |
| "step": 58 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 813.4965362548828, | |
| "epoch": 1.1339712918660287, | |
| "grad_norm": 0.08335533132880214, | |
| "kl": 0.006591796875, | |
| "learning_rate": 5.272078740225792e-07, | |
| "loss": 0.025, | |
| "reward": 0.8862847238779068, | |
| "reward_std": 0.26148218661546707, | |
| "rewards/accuracy_reward": 0.4062500149011612, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4800347238779068, | |
| "step": 59 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 799.8507232666016, | |
| "epoch": 1.1531100478468899, | |
| "grad_norm": 0.07203309192614081, | |
| "kl": 0.0055999755859375, | |
| "learning_rate": 5.1204202028063e-07, | |
| "loss": 0.0212, | |
| "reward": 0.9774305373430252, | |
| "reward_std": 0.2627098001539707, | |
| "rewards/accuracy_reward": 0.4930555745959282, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4843750149011612, | |
| "step": 60 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 791.4027862548828, | |
| "epoch": 1.1722488038277512, | |
| "grad_norm": 0.07531690542769269, | |
| "kl": 0.00594329833984375, | |
| "learning_rate": 4.969194772821193e-07, | |
| "loss": 0.0127, | |
| "reward": 0.981770858168602, | |
| "reward_std": 0.26884788647294044, | |
| "rewards/accuracy_reward": 0.486111119389534, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4956597313284874, | |
| "step": 61 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 856.4513854980469, | |
| "epoch": 1.1913875598086126, | |
| "grad_norm": 0.062172711166539586, | |
| "kl": 0.00542449951171875, | |
| "learning_rate": 4.818575001229406e-07, | |
| "loss": 0.0132, | |
| "reward": 0.8255208432674408, | |
| "reward_std": 0.20255662500858307, | |
| "rewards/accuracy_reward": 0.3819444477558136, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4435763955116272, | |
| "step": 62 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 865.0312652587891, | |
| "epoch": 1.2105263157894737, | |
| "grad_norm": 0.056253844990592314, | |
| "kl": 0.00551605224609375, | |
| "learning_rate": 4.668732747922649e-07, | |
| "loss": 0.0133, | |
| "reward": 0.767361119389534, | |
| "reward_std": 0.19641854986548424, | |
| "rewards/accuracy_reward": 0.3472222313284874, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4201389029622078, | |
| "step": 63 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 828.03125, | |
| "epoch": 1.229665071770335, | |
| "grad_norm": 0.06432066728138643, | |
| "kl": 0.005279541015625, | |
| "learning_rate": 4.5198389856301677e-07, | |
| "loss": 0.0216, | |
| "reward": 0.8559027761220932, | |
| "reward_std": 0.23324701562523842, | |
| "rewards/accuracy_reward": 0.4027777835726738, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4531250074505806, | |
| "step": 64 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 827.1354217529297, | |
| "epoch": 1.2488038277511961, | |
| "grad_norm": 0.06953232592329361, | |
| "kl": 0.005645751953125, | |
| "learning_rate": 4.3720636048357573e-07, | |
| "loss": 0.0207, | |
| "reward": 0.8515625, | |
| "reward_std": 0.20992231741547585, | |
| "rewards/accuracy_reward": 0.399305559694767, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4522569477558136, | |
| "step": 65 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 830.3020782470703, | |
| "epoch": 1.2679425837320575, | |
| "grad_norm": 0.06833410523166436, | |
| "kl": 0.00586700439453125, | |
| "learning_rate": 4.225575219929631e-07, | |
| "loss": 0.0255, | |
| "reward": 0.8984375149011612, | |
| "reward_std": 0.29340019077062607, | |
| "rewards/accuracy_reward": 0.416666679084301, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4817708283662796, | |
| "step": 66 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 837.3264007568359, | |
| "epoch": 1.2870813397129186, | |
| "grad_norm": 0.06984826024421593, | |
| "kl": 0.00540924072265625, | |
| "learning_rate": 4.0805409768163504e-07, | |
| "loss": 0.0133, | |
| "reward": 0.864583358168602, | |
| "reward_std": 0.28726211935281754, | |
| "rewards/accuracy_reward": 0.4027777910232544, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4618055522441864, | |
| "step": 67 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 813.1770782470703, | |
| "epoch": 1.30622009569378, | |
| "grad_norm": 0.07284090789531786, | |
| "kl": 0.00634765625, | |
| "learning_rate": 3.9371263621983097e-07, | |
| "loss": 0.0162, | |
| "reward": 0.8993055671453476, | |
| "reward_std": 0.27744119614362717, | |
| "rewards/accuracy_reward": 0.4270833358168602, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4722222313284874, | |
| "step": 68 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 816.5382080078125, | |
| "epoch": 1.325358851674641, | |
| "grad_norm": 0.07703787035943734, | |
| "kl": 0.0065155029296875, | |
| "learning_rate": 3.795495014752408e-07, | |
| "loss": 0.0256, | |
| "reward": 0.956597238779068, | |
| "reward_std": 0.3044487535953522, | |
| "rewards/accuracy_reward": 0.461805559694767, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4947916641831398, | |
| "step": 69 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 803.5173645019531, | |
| "epoch": 1.3444976076555024, | |
| "grad_norm": 0.07147321350786168, | |
| "kl": 0.0061187744140625, | |
| "learning_rate": 3.655808538415357e-07, | |
| "loss": 0.026, | |
| "reward": 0.9496528059244156, | |
| "reward_std": 0.2700754851102829, | |
| "rewards/accuracy_reward": 0.4687500074505806, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4809027761220932, | |
| "step": 70 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 849.3402709960938, | |
| "epoch": 1.3636363636363638, | |
| "grad_norm": 0.062268739561404315, | |
| "kl": 0.00612640380859375, | |
| "learning_rate": 3.5182263179906457e-07, | |
| "loss": 0.0233, | |
| "reward": 0.8515624850988388, | |
| "reward_std": 0.23692986741662025, | |
| "rewards/accuracy_reward": 0.3993055671453476, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4522569477558136, | |
| "step": 71 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 827.8125, | |
| "epoch": 1.3827751196172249, | |
| "grad_norm": 0.07644202407601894, | |
| "kl": 0.00571441650390625, | |
| "learning_rate": 3.3829053372876006e-07, | |
| "loss": 0.029, | |
| "reward": 0.8593750149011612, | |
| "reward_std": 0.2700754962861538, | |
| "rewards/accuracy_reward": 0.3888888955116272, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4704861119389534, | |
| "step": 72 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 824.4514007568359, | |
| "epoch": 1.401913875598086, | |
| "grad_norm": 0.07665948925496174, | |
| "kl": 0.0063323974609375, | |
| "learning_rate": 3.250000000000001e-07, | |
| "loss": 0.025, | |
| "reward": 0.9288194477558136, | |
| "reward_std": 0.2995382845401764, | |
| "rewards/accuracy_reward": 0.4375, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4913194477558136, | |
| "step": 73 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 798.7048645019531, | |
| "epoch": 1.4210526315789473, | |
| "grad_norm": 0.06940993863431441, | |
| "kl": 0.0066375732421875, | |
| "learning_rate": 3.1196619535286707e-07, | |
| "loss": 0.0166, | |
| "reward": 0.9149305522441864, | |
| "reward_std": 0.21360516920685768, | |
| "rewards/accuracy_reward": 0.423611119389534, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.491319440305233, | |
| "step": 74 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 828.0729217529297, | |
| "epoch": 1.4401913875598087, | |
| "grad_norm": 0.07296376411015054, | |
| "kl": 0.006256103515625, | |
| "learning_rate": 2.992039915949036e-07, | |
| "loss": 0.0193, | |
| "reward": 0.9097222238779068, | |
| "reward_std": 0.24552317708730698, | |
| "rewards/accuracy_reward": 0.430555559694767, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4791666716337204, | |
| "step": 75 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 816.3090362548828, | |
| "epoch": 1.4593301435406698, | |
| "grad_norm": 0.07617903452488152, | |
| "kl": 0.0065765380859375, | |
| "learning_rate": 2.867279506321115e-07, | |
| "loss": 0.017, | |
| "reward": 0.8394097238779068, | |
| "reward_std": 0.2835792824625969, | |
| "rewards/accuracy_reward": 0.3715277835726738, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4678819477558136, | |
| "step": 76 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 841.1701507568359, | |
| "epoch": 1.4784688995215312, | |
| "grad_norm": 0.0675783715098431, | |
| "kl": 0.0067291259765625, | |
| "learning_rate": 2.7455230785355164e-07, | |
| "loss": 0.0299, | |
| "reward": 0.8255208432674408, | |
| "reward_std": 0.20010139793157578, | |
| "rewards/accuracy_reward": 0.3750000074505806, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4505208358168602, | |
| "step": 77 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 863.5416717529297, | |
| "epoch": 1.4976076555023923, | |
| "grad_norm": 0.06878244801121526, | |
| "kl": 0.00676727294921875, | |
| "learning_rate": 2.6269095588850744e-07, | |
| "loss": 0.0208, | |
| "reward": 0.8307291865348816, | |
| "reward_std": 0.26148219406604767, | |
| "rewards/accuracy_reward": 0.378472238779068, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4522569552063942, | |
| "step": 78 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 827.4583435058594, | |
| "epoch": 1.5167464114832536, | |
| "grad_norm": 0.07251844117312602, | |
| "kl": 0.007293701171875, | |
| "learning_rate": 2.511574287547423e-07, | |
| "loss": 0.0209, | |
| "reward": 0.884548619389534, | |
| "reward_std": 0.281124047935009, | |
| "rewards/accuracy_reward": 0.4131944552063942, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4713541716337204, | |
| "step": 79 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 815.7916870117188, | |
| "epoch": 1.535885167464115, | |
| "grad_norm": 0.06598452519685939, | |
| "kl": 0.00701141357421875, | |
| "learning_rate": 2.39964886415941e-07, | |
| "loss": 0.0212, | |
| "reward": 0.862847238779068, | |
| "reward_std": 0.25043364614248276, | |
| "rewards/accuracy_reward": 0.399305559694767, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4635416641831398, | |
| "step": 80 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 809.65625, | |
| "epoch": 1.555023923444976, | |
| "grad_norm": 0.07534072488970928, | |
| "kl": 0.0063323974609375, | |
| "learning_rate": 2.291260997659509e-07, | |
| "loss": 0.0184, | |
| "reward": 0.9218750149011612, | |
| "reward_std": 0.28480689227581024, | |
| "rewards/accuracy_reward": 0.4340277761220932, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4878472164273262, | |
| "step": 81 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 849.3958282470703, | |
| "epoch": 1.5741626794258372, | |
| "grad_norm": 0.06432192299979189, | |
| "kl": 0.0068206787109375, | |
| "learning_rate": 2.1865343605696128e-07, | |
| "loss": 0.0169, | |
| "reward": 0.8359375149011612, | |
| "reward_std": 0.21728801727294922, | |
| "rewards/accuracy_reward": 0.3645833358168602, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4713541641831398, | |
| "step": 82 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 839.3923492431641, | |
| "epoch": 1.5933014354066986, | |
| "grad_norm": 0.0750282166543299, | |
| "kl": 0.00699615478515625, | |
| "learning_rate": 2.085588447882441e-07, | |
| "loss": 0.0295, | |
| "reward": 0.8680555671453476, | |
| "reward_std": 0.30690397322177887, | |
| "rewards/accuracy_reward": 0.3923611119389534, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.475694440305233, | |
| "step": 83 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 862.53125, | |
| "epoch": 1.61244019138756, | |
| "grad_norm": 0.0705730634901919, | |
| "kl": 0.00653839111328125, | |
| "learning_rate": 1.9885384407155872e-07, | |
| "loss": 0.0264, | |
| "reward": 0.8081597238779068, | |
| "reward_std": 0.2737583592534065, | |
| "rewards/accuracy_reward": 0.3611111119389534, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4470486119389534, | |
| "step": 84 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 832.2951354980469, | |
| "epoch": 1.631578947368421, | |
| "grad_norm": 0.07373687083533499, | |
| "kl": 0.00716400146484375, | |
| "learning_rate": 1.89549507488777e-07, | |
| "loss": 0.018, | |
| "reward": 0.9184027761220932, | |
| "reward_std": 0.22833656147122383, | |
| "rewards/accuracy_reward": 0.4305555671453476, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4878472238779068, | |
| "step": 85 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 803.6944580078125, | |
| "epoch": 1.6507177033492821, | |
| "grad_norm": 0.06861588416484331, | |
| "kl": 0.007843017578125, | |
| "learning_rate": 1.8065645145672575e-07, | |
| "loss": 0.0224, | |
| "reward": 0.9375000149011612, | |
| "reward_std": 0.24552318453788757, | |
| "rewards/accuracy_reward": 0.4513888955116272, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4861111119389534, | |
| "step": 86 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 832.5069580078125, | |
| "epoch": 1.6698564593301435, | |
| "grad_norm": 0.06855680549432348, | |
| "kl": 0.007110595703125, | |
| "learning_rate": 1.7218482311366245e-07, | |
| "loss": 0.0183, | |
| "reward": 0.8836805522441864, | |
| "reward_std": 0.2602545693516731, | |
| "rewards/accuracy_reward": 0.4131944477558136, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.470486119389534, | |
| "step": 87 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 819.6875152587891, | |
| "epoch": 1.6889952153110048, | |
| "grad_norm": 0.06513799373796432, | |
| "kl": 0.00678253173828125, | |
| "learning_rate": 1.64144288741207e-07, | |
| "loss": 0.027, | |
| "reward": 0.9244791865348816, | |
| "reward_std": 0.2590269595384598, | |
| "rewards/accuracy_reward": 0.447916679084301, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4765625, | |
| "step": 88 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 807.8333435058594, | |
| "epoch": 1.7081339712918662, | |
| "grad_norm": 0.06838857628608602, | |
| "kl": 0.00734710693359375, | |
| "learning_rate": 1.5654402273493807e-07, | |
| "loss": 0.0204, | |
| "reward": 0.9175347238779068, | |
| "reward_std": 0.2516612708568573, | |
| "rewards/accuracy_reward": 0.4409722238779068, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4765624925494194, | |
| "step": 89 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 830.8576507568359, | |
| "epoch": 1.7272727272727273, | |
| "grad_norm": 0.06707500592677755, | |
| "kl": 0.00732421875, | |
| "learning_rate": 1.4939269713624181e-07, | |
| "loss": 0.0171, | |
| "reward": 0.9131944626569748, | |
| "reward_std": 0.22833655402064323, | |
| "rewards/accuracy_reward": 0.4409722313284874, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4722222238779068, | |
| "step": 90 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 805.3263854980469, | |
| "epoch": 1.7464114832535884, | |
| "grad_norm": 0.08133209124657947, | |
| "kl": 0.007537841796875, | |
| "learning_rate": 1.4269847173735543e-07, | |
| "loss": 0.0284, | |
| "reward": 0.942708358168602, | |
| "reward_std": 0.27989643067121506, | |
| "rewards/accuracy_reward": 0.4409722238779068, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.5017361044883728, | |
| "step": 91 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 833.0833282470703, | |
| "epoch": 1.7655502392344498, | |
| "grad_norm": 0.06350369442926171, | |
| "kl": 0.00672149658203125, | |
| "learning_rate": 1.3646898477089625e-07, | |
| "loss": 0.0197, | |
| "reward": 0.8750000149011612, | |
| "reward_std": 0.24306794628500938, | |
| "rewards/accuracy_reward": 0.4131944477558136, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.461805559694767, | |
| "step": 92 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 840.3715362548828, | |
| "epoch": 1.784688995215311, | |
| "grad_norm": 0.07310106942013637, | |
| "kl": 0.00698089599609375, | |
| "learning_rate": 1.307113441944992e-07, | |
| "loss": 0.0204, | |
| "reward": 0.8324652761220932, | |
| "reward_std": 0.26639265939593315, | |
| "rewards/accuracy_reward": 0.3750000074505806, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4574652835726738, | |
| "step": 93 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 798.3472290039062, | |
| "epoch": 1.8038277511961722, | |
| "grad_norm": 0.07470777769552865, | |
| "kl": 0.0069580078125, | |
| "learning_rate": 1.2543211958050894e-07, | |
| "loss": 0.0225, | |
| "reward": 0.9123263955116272, | |
| "reward_std": 0.26884787529706955, | |
| "rewards/accuracy_reward": 0.4409722238779068, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4713541641831398, | |
| "step": 94 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 809.6944580078125, | |
| "epoch": 1.8229665071770333, | |
| "grad_norm": 0.06571441072063877, | |
| "kl": 0.007171630859375, | |
| "learning_rate": 1.2063733461997803e-07, | |
| "loss": 0.0208, | |
| "reward": 0.9305555671453476, | |
| "reward_std": 0.21606039628386497, | |
| "rewards/accuracy_reward": 0.447916679084301, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4826388955116272, | |
| "step": 95 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 814.4757080078125, | |
| "epoch": 1.8421052631578947, | |
| "grad_norm": 0.07474184052328098, | |
| "kl": 0.0072021484375, | |
| "learning_rate": 1.1633246024952662e-07, | |
| "loss": 0.0291, | |
| "reward": 0.8836805522441864, | |
| "reward_std": 0.2602545730769634, | |
| "rewards/accuracy_reward": 0.40625, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.477430559694767, | |
| "step": 96 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 817.5972290039062, | |
| "epoch": 1.861244019138756, | |
| "grad_norm": 0.08343014010656753, | |
| "kl": 0.00762939453125, | |
| "learning_rate": 1.1252240840890425e-07, | |
| "loss": 0.0248, | |
| "reward": 0.9192708283662796, | |
| "reward_std": 0.27130311727523804, | |
| "rewards/accuracy_reward": 0.4375000149011612, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4817708358168602, | |
| "step": 97 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 812.0278015136719, | |
| "epoch": 1.8803827751196174, | |
| "grad_norm": 0.07184568208672217, | |
| "kl": 0.00701904296875, | |
| "learning_rate": 1.0921152643637749e-07, | |
| "loss": 0.0289, | |
| "reward": 0.90625, | |
| "reward_std": 0.2725307270884514, | |
| "rewards/accuracy_reward": 0.4305555522441864, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4756944477558136, | |
| "step": 98 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 780.3958435058594, | |
| "epoch": 1.8995215311004785, | |
| "grad_norm": 0.06820176876725473, | |
| "kl": 0.0081787109375, | |
| "learning_rate": 1.064035921083383e-07, | |
| "loss": 0.0215, | |
| "reward": 0.9201388955116272, | |
| "reward_std": 0.22588132694363594, | |
| "rewards/accuracy_reward": 0.430555559694767, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4895833432674408, | |
| "step": 99 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 812.1562652587891, | |
| "epoch": 1.9186602870813396, | |
| "grad_norm": 0.07452483871871766, | |
| "kl": 0.00782012939453125, | |
| "learning_rate": 1.0410180932879252e-07, | |
| "loss": 0.0291, | |
| "reward": 0.9079861044883728, | |
| "reward_std": 0.31181444972753525, | |
| "rewards/accuracy_reward": 0.4201388880610466, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4878472313284874, | |
| "step": 100 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 827.5972442626953, | |
| "epoch": 1.937799043062201, | |
| "grad_norm": 0.06920787242163999, | |
| "kl": 0.00714874267578125, | |
| "learning_rate": 1.0230880447364718e-07, | |
| "loss": 0.0181, | |
| "reward": 0.8758680671453476, | |
| "reward_std": 0.2762135788798332, | |
| "rewards/accuracy_reward": 0.4027777835726738, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4730902835726738, | |
| "step": 101 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 848.9201354980469, | |
| "epoch": 1.9569377990430623, | |
| "grad_norm": 0.06359574498753441, | |
| "kl": 0.00762939453125, | |
| "learning_rate": 1.0102662339396817e-07, | |
| "loss": 0.0237, | |
| "reward": 0.8289930522441864, | |
| "reward_std": 0.23938510566949844, | |
| "rewards/accuracy_reward": 0.3611111119389534, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4678819552063942, | |
| "step": 102 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 827.0173645019531, | |
| "epoch": 1.9760765550239234, | |
| "grad_norm": 0.06784856725869416, | |
| "kl": 0.00757598876953125, | |
| "learning_rate": 1.002567290816269e-07, | |
| "loss": 0.0233, | |
| "reward": 0.9071180671453476, | |
| "reward_std": 0.2565717250108719, | |
| "rewards/accuracy_reward": 0.4340277761220932, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4730902835726738, | |
| "step": 103 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 824.1666870117188, | |
| "epoch": 1.9952153110047846, | |
| "grad_norm": 0.07003424025971802, | |
| "kl": 0.00720977783203125, | |
| "learning_rate": 1e-07, | |
| "loss": 0.0232, | |
| "reward": 0.8446180522441864, | |
| "reward_std": 0.24920602142810822, | |
| "rewards/accuracy_reward": 0.3784722313284874, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4661458358168602, | |
| "step": 104 | |
| }, | |
| { | |
| "epoch": 1.9952153110047846, | |
| "step": 104, | |
| "total_flos": 0.0, | |
| "train_loss": 0.02088056951134394, | |
| "train_runtime": 4550.0212, | |
| "train_samples_per_second": 3.297, | |
| "train_steps_per_second": 0.023 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 104, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 2, | |
| "save_steps": 10, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 24, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |