Model save

README.md

@@ -1,11 +1,9 @@
 ---
-base_model: deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B
-datasets: knoveleng/open-rs
+base_model: Qwen/Qwen3.5-0.8B
 library_name: transformers
 model_name: OpenRS-GRPO
 tags:
 - generated_from_trainer
-- open-r1
 - trl
 - grpo
 licence: license
@@ -13,7 +11,7 @@ licence: license
 
 # Model Card for OpenRS-GRPO
 
-This model is a fine-tuned version of [deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B) on the [knoveleng/open-rs](https://huggingface.co/datasets/knoveleng/open-rs) dataset.
+This model is a fine-tuned version of [Qwen/Qwen3.5-0.8B](https://huggingface.co/Qwen/Qwen3.5-0.8B).
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0013520326372236013,
-    "train_runtime": 36067.9715,
+    "train_loss": 0.010430806130170823,
+    "train_runtime": 9458.0577,
     "train_samples": 7000,
-    "train_samples_per_second": 0.044,
-    "train_steps_per_second": 0.003
+    "train_samples_per_second": 0.034,
+    "train_steps_per_second": 0.002
 }

step_metrics.csv

@@ -1,102 +1,22 @@
 step,epoch,loss,learning_rate,grad_norm,rewards/format_reward,rewards/cosine_scaled_reward,reward,reward_std,gpu_mem_alloc_mb,gpu_mem_peak_mb,step_time_sec
-1,0.0006,3.3527612686157227e-08,0.0,,0.0,-0.45186813920736313,-0.9037362784147263,0.11632500402629375,3617.5,6086.5,356.01
-2,0.0011,-2.9802322387695312e-08,0.0,,0.0,-0.4544064328074455,-0.9088128805160522,0.134795643389225,3617.5,6091.3,357.14
-3,0.0017,-2.384185791015625e-07,0.0,,0.0,-0.44806819409132004,-0.8961364179849625,0.1266492954455316,3617.5,6092.8,356.69
-4,0.0023,3.203749656677246e-07,0.0,,0.0,-0.46924517303705215,-0.9384903311729431,0.08165389159694314,3617.5,6095.2,358.13
-5,0.0029,2.980232238769531e-07,0.0,,0.0,-0.4361194893717766,-0.8722389936447144,0.1861576708033681,3617.5,6095.2,355.88
-6,0.0034,3.2782554626464844e-07,0.0,,0.0,-0.4838990569114685,-0.9677980989217758,0.03695695102214813,3617.5,6102.0,359.97
-7,0.004,-6.48200511932373e-07,0.0,,0.0,-0.46913372725248337,-0.9382674694061279,0.04741490981541574,3617.5,6102.0,366.73
-8,0.0046,-1.9371509552001953e-07,0.0,,0.0,-0.4689921736717224,-0.9379843473434448,0.08557501714676619,3617.5,6102.0,359.73
-9,0.0051,6.258487701416016e-07,0.0,,0.0,-0.47829224169254303,-0.9565844535827637,0.04161944845691323,3617.5,6102.0,356.95
-10,0.0057,0.0,0.0,,0.0,-0.39301924407482147,-0.7860384881496429,0.29622524976730347,3654.4,6102.0,356.9
-11,0.0063,-1.2665987014770508e-06,1.0000000000000002e-06,,0.0,-0.47354312986135483,-0.9470862597227097,0.025707244174554944,3654.4,6129.7,357.65
-12,0.0069,-3.2782554626464844e-07,1.0000000000000002e-06,,0.0,-0.4611263796687126,-0.9222527593374252,0.11853919085115194,3654.4,6132.1,359.0
-13,0.0074,-9.834766387939453e-07,2.0000000000000003e-06,,0.0,-0.40241140127182007,-0.8048228025436401,0.2713719364255667,3654.4,6132.1,359.35
-14,0.008,-9.98377799987793e-07,3e-06,,0.0,-0.4491143301129341,-0.8982286602258682,0.15008432138711214,3654.4,6132.1,359.42
-15,0.0086,-1.2367963790893555e-06,4.000000000000001e-06,,0.0,-0.4349117949604988,-0.8698235750198364,0.1454296549782157,3654.4,6132.1,358.18
-16,0.0091,-1.166015863418579e-06,5e-06,,0.0,-0.4481472671031952,-0.8962945342063904,0.11415091808885336,3654.4,6136.0,359.05
-17,0.0097,-4.172325134277344e-07,6e-06,,0.0,-0.40375038236379623,-0.8075007796287537,0.2773375315591693,3654.4,6136.0,359.81
-18,0.0103,1.0356307029724121e-06,7e-06,,0.0,-0.4421778917312622,-0.8843557834625244,0.1459937175968662,3654.4,6136.0,361.88
-19,0.0109,-1.4901161193847656e-08,8.000000000000001e-06,,0.0,-0.46627073734998703,-0.9325414896011353,0.061190704349428415,3654.4,6136.0,361.95
-20,0.0114,7.450580596923828e-08,9e-06,,0.0,-0.4463533014059067,-0.8927065879106522,0.1468478236347437,3654.4,6136.0,361.89
-21,0.012,-3.241002559661865e-07,1e-05,,0.0,-0.430856853723526,-0.861713707447052,0.15973031288012862,3654.4,6136.0,360.43
-22,0.0126,0.0,9.997258721585931e-06,,0.0,-0.42839662730693817,-0.8567932546138763,0.23303062841296196,3654.4,6136.0,359.97
-23,0.0131,-1.9371509552001953e-07,9.98903822616921e-06,,0.0,-0.45275644212961197,-0.9055128693580627,0.13258774112910032,3654.4,6136.0,360.57
-24,0.0137,4.3958425521850586e-07,9.97534852915723e-06,,0.0,-0.46590057015419006,-0.9318011105060577,0.08933348534628749,3654.4,6136.0,362.48
-25,0.0143,-8.940696716308594e-08,9.956206309337067e-06,,0.0,-0.45125550776720047,-0.9025110006332397,0.15539621422067285,3654.4,6136.0,364.1
-26,0.0149,-2.8312206268310547e-07,9.931634888554937e-06,,0.0,-0.456791490316391,-0.9135829955339432,0.08904895093291998,3654.4,6136.0,365.75
-27,0.0154,-2.60770320892334e-06,9.901664203302126e-06,,0.0,-0.4751305654644966,-0.950261116027832,0.06268075766274706,3654.4,6136.0,370.26
-28,0.016,5.960464477539063e-08,9.866330768241984e-06,,0.0,-0.4367612153291702,-0.8735224008560181,0.16378713678568602,3654.4,6138.0,366.74
-29,0.0166,4.842877388000488e-07,9.825677631722436e-06,,0.0,-0.4282456487417221,-0.856491282582283,0.1944181639701128,3654.5,6138.0,364.94
-30,0.0171,1.0058283805847168e-06,9.779754323328192e-06,,0.0,-0.45244697481393814,-0.9048939347267151,0.1466370872221887,3654.4,6138.0,359.72
-31,0.0177,7.301568984985352e-07,9.728616793536588e-06,,0.0,-0.4787774831056595,-0.9575549513101578,0.04152237856760621,3654.4,6138.0,359.74
-32,0.0183,7.525086402893066e-07,9.672327345550544e-06,,0.0,-0.4640432074666023,-0.9280864149332047,0.05714223568793386,3654.4,6138.0,359.53
-33,0.0189,6.705522537231445e-07,9.610954559391704e-06,,0.0,-0.42499294877052307,-0.8499859273433685,0.13816553819924593,3654.5,6138.0,362.56
-34,0.0194,3.4868717193603516e-06,9.544573208346252e-06,,0.0,-0.4555760398507118,-0.9111520648002625,0.06401598325464875,3654.4,6138.0,362.18
-35,0.02,1.996755599975586e-06,9.473264167865172e-06,,0.0,-0.4616549611091614,-0.9233099222183228,0.07124835508875549,3654.4,6138.0,360.07
-36,0.0206,3.5315752029418945e-06,9.397114317029975e-06,,0.0,-0.43700823187828064,-0.8740164637565613,0.1880413582548499,3654.4,6138.0,358.61
-37,0.0211,0.1341422200202942,9.316216432703918e-06,,0.0,-0.40694746375083923,-0.8138948976993561,0.2712679710239172,3654.4,6138.0,357.92
-38,0.0217,4.366040229797363e-06,9.230669076497688e-06,,0.0,-0.4537286013364792,-0.9074572026729584,0.140884583350271,3654.5,6138.0,361.01
-39,0.0223,2.9802322387695312e-06,9.140576474687263e-06,,0.0,-0.4670984223484993,-0.9341968446969986,0.08859914634376764,3654.4,6138.0,358.53
-40,0.0229,3.2633543014526367e-06,9.046048391230248e-06,,0.0,-0.4738834798336029,-0.9477669596672058,0.03630512161180377,3654.4,6138.0,356.31
-41,0.0234,3.5390257835388184e-06,8.947199994035402e-06,,0.0,-0.48112839460372925,-0.9622567743062973,0.03198406333103776,3654.4,6138.0,356.46
-42,0.024,4.059635102748871e-06,8.844151714648274e-06,,0.0,-0.45688286423683167,-0.9137657284736633,0.09395218873396516,3654.4,6138.0,358.44
-43,0.0246,3.972090780735016e-06,8.737029101523931e-06,,0.0,-0.4356464073061943,-0.8712927997112274,0.17926698923110962,3654.4,6138.0,362.88
-44,0.0251,7.338821887969971e-06,8.625962667065488e-06,,0.0,-0.4162614122033119,-0.8325228244066238,0.22310001868754625,3654.4,6138.0,358.73
-45,0.0257,9.98377799987793e-06,8.511087728614863e-06,,0.0,-0.4377835765480995,-0.8755671381950378,0.1843216335400939,3654.4,6138.0,359.1
-46,0.0263,7.063150405883789e-06,8.392544243589428e-06,,0.0,-0.4647279307246208,-0.9294558465480804,0.08181617665104568,3654.4,6138.0,364.32
-47,0.0269,9.715557098388672e-06,8.270476638965463e-06,,0.0,-0.4818853959441185,-0.9637707620859146,0.04111184738576412,3654.4,6138.0,366.26
-48,0.0274,1.0699033737182617e-05,8.14503363531613e-06,,0.0,-0.47147445380687714,-0.9429489076137543,0.05583410756662488,3654.4,6138.0,367.58
-49,0.028,9.488314390182495e-06,8.016368065618361e-06,,0.0,-0.4839174821972847,-0.9678349643945694,0.031033652368932962,3654.4,6138.0,367.03
-50,0.0286,1.1995434761047363e-05,7.884636689049423e-06,,0.0,-0.4634588584303856,-0.92691770195961,0.06057529430836439,3654.4,6138.0,365.37
-51,0.0291,1.093745231628418e-05,7.75e-06,,0.0,-0.48549777269363403,-0.9709955900907516,0.023253681138157845,3654.5,6138.0,357.96
-52,0.0297,1.2755393981933594e-05,7.612622032536508e-06,,0.0,-0.4660480171442032,-0.932096004486084,0.0814181575551629,3654.5,6138.0,357.69
-53,0.0303,1.1809170246124268e-05,7.472670160550849e-06,,0.0,-0.42992351949214935,-0.8598470240831375,0.23317514825612307,3654.5,6138.0,360.51
-54,0.0309,1.2435019016265869e-05,7.330314893841102e-06,,0.0,-0.4458845555782318,-0.8917691111564636,0.16836319211870432,3654.5,6138.0,359.99
-55,0.0314,1.2367963790893555e-05,7.185729670371605e-06,,0.0,-0.46969927847385406,-0.9393985569477081,0.09125666646286845,3654.5,6138.0,357.92
-56,0.032,1.3027340173721313e-05,7.0390906449655104e-06,,0.0,-0.4778226688504219,-0.9556453377008438,0.03563447529450059,3654.4,6138.0,359.54
-57,0.0326,1.173466444015503e-05,6.890576474687264e-06,,0.0,-0.4766504615545273,-0.9533008933067322,0.047348865773528814,3654.5,6138.0,360.89
-58,0.0331,1.0944902896881104e-05,6.740368101176496e-06,,0.0,-0.4303411394357681,-0.8606822788715363,0.19956759549677372,3654.5,6138.0,357.96
-59,0.0337,1.4327466487884521e-05,6.588648530198505e-06,,0.0,-0.47175391018390656,-0.9435078203678131,0.028082083677873015,3654.4,6138.0,358.32
-60,0.0343,1.862645149230957e-05,6.4356026086799176e-06,,0.0,-0.4349738284945488,-0.8699476420879364,0.14623272977769375,3654.5,6138.0,358.57
-61,0.0349,1.6495585441589355e-05,6.281416799501188e-06,,0.0,-0.43933914601802826,-0.8786782920360565,0.14873503288254142,3654.5,6138.0,359.81
-62,0.0354,1.4573335647583008e-05,6.126278954320295e-06,,0.0,-0.47655102610588074,-0.9531020373106003,0.04140742728486657,3654.5,6138.0,363.55
-63,0.036,1.3574957847595215e-05,5.970378084704441e-06,,0.0,-0.4812774509191513,-0.9625549018383026,0.0323515310883522,3654.4,6138.0,361.67
-64,0.0366,1.7490237951278687e-05,5.813904131848565e-06,,0.0,-0.4819137006998062,-0.9638274163007736,0.03202465921640396,3654.5,6138.0,360.66
-65,0.0371,1.4990568161010742e-05,5.657047735161256e-06,,0.0,-0.4426536113023758,-0.8853072375059128,0.11870932951569557,3654.5,6138.0,360.47
-66,0.0377,1.6763806343078613e-05,5.500000000000001e-06,,0.0,-0.4815715327858925,-0.9631430506706238,0.027665999252349138,3654.4,6138.0,364.95
-67,0.0383,2.3305416107177734e-05,5.342952264838748e-06,,0.0,-0.46308349817991257,-0.9261669814586639,0.09819867718033493,3654.4,6138.0,369.54
-68,0.0389,2.4430453777313232e-05,5.186095868151436e-06,,0.0,-0.4661281928420067,-0.9322563856840134,0.06839151354506612,3654.4,6138.0,369.64
-69,0.0394,1.753866672515869e-05,5.02962191529556e-06,,0.0,-0.4736683666706085,-0.947336733341217,0.06602911371737719,3654.5,6138.0,365.41
-70,0.04,1.9043684005737305e-05,4.873721045679707e-06,,0.0,-0.440708264708519,-0.881416529417038,0.15695088542997837,3654.4,6138.0,364.8
-71,0.0406,2.4437904357910156e-05,4.718583200498814e-06,,0.0,-0.47764309495687485,-0.9552861750125885,0.04716450162231922,3654.4,6138.0,362.55
-72,0.0411,1.781061291694641e-05,4.564397391320085e-06,,0.0,-0.45879723131656647,-0.9175944626331329,0.07855925057083368,3654.4,6138.0,359.86
-73,0.0417,2.104230225086212e-05,4.4113514698014955e-06,,0.0,-0.47955460846424103,-0.9591091871261597,0.03032594360411167,3654.4,6138.0,354.97
-74,0.0423,1.8224120140075684e-05,4.259631898823504e-06,,0.0,-0.470923513174057,-0.9418470114469528,0.09771440364420414,3654.4,6138.0,359.71
-75,0.0429,1.697242259979248e-05,4.109423525312738e-06,,0.0,-0.4515884444117546,-0.9031769037246704,0.11431426065973938,3654.4,6138.0,359.51
-76,0.0434,1.5683472156524658e-05,3.960909355034491e-06,,0.0,-0.43553559482097626,-0.8710711896419525,0.16496195830404758,3654.4,6138.0,357.76
-77,0.044,2.2258609533309937e-05,3.8142703296283954e-06,,0.0,-0.418168805539608,-0.8363375961780548,0.23763815127313137,3654.4,6138.0,359.55
-78,0.0446,2.06679105758667e-05,3.6696851061589e-06,,0.0,-0.45626621693372726,-0.912532389163971,0.1374040930531919,3654.5,6138.0,359.56
-79,0.0451,3.053247928619385e-05,3.527329839449152e-06,,0.0,-0.478363037109375,-0.9567261040210724,0.03342599933966994,3654.4,6138.0,359.31
-80,0.0457,3.489106893539429e-05,3.3873779674634932e-06,,0.0,-0.43451904505491257,-0.8690381050109863,0.166263896971941,3654.5,6138.0,358.11
-81,0.0463,2.2970139980316162e-05,3.2500000000000015e-06,,0.0,-0.4851565733551979,-0.9703131467103958,0.018532322952523828,3654.4,6138.0,359.52
-82,0.0469,2.2605061531066895e-05,3.115363310950579e-06,,0.0,-0.4188460633158684,-0.8376921266317368,0.21680933889001608,3654.5,6138.0,360.43
-83,0.0474,2.580881118774414e-05,2.98363193438164e-06,,0.0,-0.45973849296569824,-0.9194770008325577,0.07644858444109559,3654.4,6138.0,357.84
-84,0.048,2.232193946838379e-05,2.854966364683872e-06,,0.0,-0.4607899561524391,-0.921579897403717,0.07768060895614326,3654.4,6138.0,364.25
-85,0.0486,2.230703830718994e-05,2.7295233610345384e-06,,0.0,-0.40651462972164154,-0.8130292594432831,0.21775285061448812,3654.4,6138.0,361.8
-86,0.0491,2.0213425159454346e-05,2.607455756410573e-06,,0.0,-0.4558562785387039,-0.911712571978569,0.09181239921599627,3654.5,6138.0,363.18
-87,0.0497,2.09808349609375e-05,2.4889122713851397e-06,,0.0,-0.47007501125335693,-0.9401500076055527,0.04285714589059353,3654.5,6138.0,368.87
-88,0.0503,2.3573637008666992e-05,2.374037332934512e-06,,0.0,-0.41688134521245956,-0.8337626904249191,0.24908871483057737,3654.4,6138.0,360.15
-89,0.0509,2.3312866687774658e-05,2.262970898476071e-06,,0.0,-0.4709980934858322,-0.9419961720705032,0.06526243314146996,3654.5,6138.0,355.62
-90,0.0514,2.16066837310791e-05,2.1558482853517257e-06,,0.0,-0.47104664146900177,-0.9420932680368423,0.05442978721112013,3654.4,6138.0,359.25
-91,0.052,2.3186206817626953e-05,2.0528000059646e-06,,0.0,-0.42244888097047806,-0.8448977470397949,0.17764843348413706,3654.4,6138.0,357.58
-92,0.0526,2.4143606424331665e-05,1.953951608769752e-06,,0.0,-0.43437784910202026,-0.8687557131052017,0.13139949878677726,3654.5,6138.0,360.98
-93,0.0531,1.7457641661167145e-05,1.8594235253127373e-06,,0.0,-0.4726478382945061,-0.9452957063913345,0.044932478107512,3654.4,6138.0,359.65
-94,0.0537,2.3424625396728516e-05,1.769330923502313e-06,,0.0,-0.48662828654050827,-0.9732565432786942,0.023667596746236086,3654.4,6138.0,357.33
-95,0.0543,2.664327621459961e-05,1.6837835672960834e-06,,0.0,-0.41473422944545746,-0.8294684588909149,0.21260959655046463,3654.4,6138.0,360.81
-96,0.0549,2.310052514076233e-05,1.602885682970026e-06,,0.0,-0.4408787190914154,-0.8817574381828308,0.14791762363165617,3654.4,6138.0,359.17
-97,0.0554,1.5437602996826172e-05,1.526735832134829e-06,,0.0,-0.46087589859962463,-0.9217518121004105,0.05548063712194562,3654.5,6138.0,357.98
-98,0.056,2.017989754676819e-05,1.4554267916537495e-06,,0.0,-0.4707222431898117,-0.9414444863796234,0.057337059173732996,3654.4,6138.0,359.07
-99,0.0566,1.6748905181884766e-05,1.389045440608296e-06,,0.0,-0.43424008786678314,-0.8684801608324051,0.18490357510745525,3654.4,6138.0,357.48
-100,0.0571,2.2009015083312988e-05,1.3276726544494572e-06,,0.0,-0.44838932156562805,-0.8967786431312561,0.12791539868339896,3654.4,6138.0,358.57
-100,0.0571,,,,,,,,3654.4,6138.0,362.18
+1,0.0006,-1.2665987014770508e-07,0.0,,0.0,-0.3879377990961075,-0.7758755832910538,0.2861072635278106,1565.2,3646.7,476.39
+2,0.0011,2.2351741790771484e-08,5e-07,,0.0,-0.34302495419979095,-0.6860499083995819,0.3919920399785042,1565.2,3663.9,469.44
+3,0.0017,0.04199279844760895,1e-06,,0.0,-0.39361898601055145,-0.7872379571199417,0.28899660520255566,1565.2,3665.2,464.19
+4,0.0023,0.00015985965728759766,9.931634888554935e-07,,0.0,-0.39139123260974884,-0.7827824652194977,0.30724803544580936,1565.2,3665.5,473.28
+5,0.0029,0.016923315823078156,9.728616793536587e-07,,0.0,-0.3417773097753525,-0.6835546344518661,0.4394468888640404,1565.2,3666.6,475.57
+6,0.0034,0.00023673847317695618,9.397114317029974e-07,,0.0,-0.43544115871191025,-0.8708823472261429,0.1456776731647551,1565.2,3669.9,476.37
+7,0.004,5.0827860832214355e-05,8.9471999940354e-07,,0.0,-0.46910375356674194,-0.9382074922323227,0.08606540504842997,1565.2,3669.9,481.67
+8,0.0046,7.636845111846924e-05,8.392544243589427e-07,,0.0,-0.3953063264489174,-0.7906126528978348,0.18317685835063457,1565.2,3669.9,481.42
+9,0.0051,7.21365213394165e-05,7.75e-07,,0.0,-0.4495018497109413,-0.8990036994218826,0.1606585686095059,1565.2,3669.9,479.75
+10,0.0057,0.00016715750098228455,7.039090644965509e-07,,0.0,-0.4300354793667793,-0.860070988535881,0.17052607703953981,1565.2,3669.9,466.26
+11,0.0063,0.00013599544763565063,6.281416799501187e-07,,0.0,-0.4115590825676918,-0.8231181800365448,0.1259385095909238,1565.2,3669.9,472.08
+12,0.0069,5.13419508934021e-05,5.5e-07,,0.0,-0.4723722040653229,-0.9447444081306458,0.08061030774842948,1565.2,3669.9,471.88
+13,0.0074,0.07869705557823181,4.7185832004988133e-07,,0.0,-0.44672856479883194,-0.8934571295976639,0.15774485282599926,1565.2,3669.9,466.09
+14,0.008,0.027497582137584686,3.9609093550344907e-07,,0.0,-0.3920762911438942,-0.7841525673866272,0.2220854666084051,1565.2,3669.9,468.75
+15,0.0086,0.0004888176918029785,3.250000000000001e-07,,0.0,-0.3608057275414467,-0.7216114401817322,0.3453192347660661,1565.2,3669.9,462.49
+16,0.0091,0.00015526264905929565,2.6074557564105724e-07,,0.0,-0.4132692217826843,-0.8265384286642075,0.25778803089633584,1565.2,3669.9,476.11
+17,0.0097,0.02472507953643799,2.0528000059645995e-07,,0.0,-0.4350534752011299,-0.8701069504022598,0.18937412789091468,1565.2,3669.9,469.71
+18,0.0103,0.00023746490478515625,1.6028856829700258e-07,,0.0,-0.41003918647766113,-0.8200783580541611,0.26157089229673147,1565.2,3669.9,472.28
+19,0.0109,0.016675502061843872,1.2713832064634125e-07,,0.0,-0.41168487817049026,-0.8233697563409805,0.20013628248125315,1565.2,3669.9,477.29
+20,0.0114,0.00027292221784591675,1.068365111445064e-07,,0.0,-0.4022079259157181,-0.8044158518314362,0.2423506089253351,1565.2,3669.9,473.18
+20,0.0114,,,,,,,,1565.2,3669.9,477.0

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0013520326372236013,
-    "train_runtime": 36067.9715,
+    "train_loss": 0.010430806130170823,
+    "train_runtime": 9458.0577,
     "train_samples": 7000,
-    "train_samples_per_second": 0.044,
-    "train_steps_per_second": 0.003
+    "train_samples_per_second": 0.034,
+    "train_steps_per_second": 0.002
 }

trainer_state.json

@@ -2,1325 +2,285 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.05714285714285714,
+  "epoch": 0.011428571428571429,
   "eval_steps": 500,
-  "global_step": 100,
+  "global_step": 20,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "clip_ratio": 0.0,
-      "completion_length": 1024.0,
+      "completion_length": 453.625,
       "epoch": 0.0005714285714285715,
       "kl": 0.0,
       "learning_rate": 0.0,
-      "loss": 3.3527612686157227e-08,
-      "reward": -0.9037362784147263,
-      "reward_std": 0.11632500402629375,
-      "rewards/cosine_scaled_reward": -0.45186813920736313,
+      "loss": -1.2665987014770508e-07,
+      "reward": -0.7758755832910538,
+      "reward_std": 0.2861072635278106,
+      "rewards/cosine_scaled_reward": -0.3879377990961075,
       "rewards/format_reward": 0.0,
       "step": 1
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 1024.0,
+      "completion_length": 512.0,
       "epoch": 0.001142857142857143,
       "kl": 0.0,
-      "learning_rate": 0.0,
-      "loss": -2.9802322387695312e-08,
-      "reward": -0.9088128805160522,
-      "reward_std": 0.134795643389225,
-      "rewards/cosine_scaled_reward": -0.4544064328074455,
+      "learning_rate": 5e-07,
+      "loss": 2.2351741790771484e-08,
+      "reward": -0.6860499083995819,
+      "reward_std": 0.3919920399785042,
+      "rewards/cosine_scaled_reward": -0.34302495419979095,
       "rewards/format_reward": 0.0,
       "step": 2
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 1024.0,
+      "completion_length": 494.9375,
       "epoch": 0.0017142857142857142,
-      "kl": 0.0,
-      "learning_rate": 0.0,
-      "loss": -2.384185791015625e-07,
-      "reward": -0.8961364179849625,
-      "reward_std": 0.1266492954455316,
-      "rewards/cosine_scaled_reward": -0.44806819409132004,
+      "kl": 0.00760650634765625,
+      "learning_rate": 1e-06,
+      "loss": 0.04199279844760895,
+      "reward": -0.7872379571199417,
+      "reward_std": 0.28899660520255566,
+      "rewards/cosine_scaled_reward": -0.39361898601055145,
       "rewards/format_reward": 0.0,
       "step": 3
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 1024.0,
+      "completion_length": 484.875,
       "epoch": 0.002285714285714286,
-      "kl": 0.0,
-      "learning_rate": 0.0,
-      "loss": 3.203749656677246e-07,
-      "reward": -0.9384903311729431,
-      "reward_std": 0.08165389159694314,
-      "rewards/cosine_scaled_reward": -0.46924517303705215,
+      "kl": 0.00399017333984375,
+      "learning_rate": 9.931634888554935e-07,
+      "loss": 0.00015985965728759766,
+      "reward": -0.7827824652194977,
+      "reward_std": 0.30724803544580936,
+      "rewards/cosine_scaled_reward": -0.39139123260974884,
       "rewards/format_reward": 0.0,
       "step": 4
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 1024.0,
+      "completion_length": 498.125,
       "epoch": 0.002857142857142857,
-      "kl": 0.0,
-      "learning_rate": 0.0,
-      "loss": 2.980232238769531e-07,
-      "reward": -0.8722389936447144,
-      "reward_std": 0.1861576708033681,
-      "rewards/cosine_scaled_reward": -0.4361194893717766,
+      "kl": 0.010009765625,
+      "learning_rate": 9.728616793536587e-07,
+      "loss": 0.016923315823078156,
+      "reward": -0.6835546344518661,
+      "reward_std": 0.4394468888640404,
+      "rewards/cosine_scaled_reward": -0.3417773097753525,
       "rewards/format_reward": 0.0,
       "step": 5
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 1024.0,
+      "completion_length": 512.0,
       "epoch": 0.0034285714285714284,
-      "kl": 0.0,
-      "learning_rate": 0.0,
-      "loss": 3.2782554626464844e-07,
-      "reward": -0.9677980989217758,
-      "reward_std": 0.03695695102214813,
-      "rewards/cosine_scaled_reward": -0.4838990569114685,
+      "kl": 0.0059185028076171875,
+      "learning_rate": 9.397114317029974e-07,
+      "loss": 0.00023673847317695618,
+      "reward": -0.8708823472261429,
+      "reward_std": 0.1456776731647551,
+      "rewards/cosine_scaled_reward": -0.43544115871191025,
       "rewards/format_reward": 0.0,
       "step": 6
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 1024.0,
+      "completion_length": 512.0,
       "epoch": 0.004,
-      "kl": 0.0,
-      "learning_rate": 0.0,
-      "loss": -6.48200511932373e-07,
-      "reward": -0.9382674694061279,
-      "reward_std": 0.04741490981541574,
-      "rewards/cosine_scaled_reward": -0.46913372725248337,
+      "kl": 0.001270294189453125,
+      "learning_rate": 8.9471999940354e-07,
+      "loss": 5.0827860832214355e-05,
+      "reward": -0.9382074922323227,
+      "reward_std": 0.08606540504842997,
+      "rewards/cosine_scaled_reward": -0.46910375356674194,
       "rewards/format_reward": 0.0,
       "step": 7
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 1024.0,
+      "completion_length": 480.625,
       "epoch": 0.004571428571428572,
-      "kl": 0.0,
-      "learning_rate": 0.0,
-      "loss": -1.9371509552001953e-07,
-      "reward": -0.9379843473434448,
-      "reward_std": 0.08557501714676619,
-      "rewards/cosine_scaled_reward": -0.4689921736717224,
+      "kl": 0.0019054412841796875,
+      "learning_rate": 8.392544243589427e-07,
+      "loss": 7.636845111846924e-05,
+      "reward": -0.7906126528978348,
+      "reward_std": 0.18317685835063457,
+      "rewards/cosine_scaled_reward": -0.3953063264489174,
       "rewards/format_reward": 0.0,
       "step": 8
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 1024.0,
+      "completion_length": 512.0,
       "epoch": 0.005142857142857143,
-      "kl": 0.0,
-      "learning_rate": 0.0,
-      "loss": 6.258487701416016e-07,
-      "reward": -0.9565844535827637,
-      "reward_std": 0.04161944845691323,
-      "rewards/cosine_scaled_reward": -0.47829224169254303,
+      "kl": 0.0017995834350585938,
+      "learning_rate": 7.75e-07,
+      "loss": 7.21365213394165e-05,
+      "reward": -0.8990036994218826,
+      "reward_std": 0.1606585686095059,
+      "rewards/cosine_scaled_reward": -0.4495018497109413,
       "rewards/format_reward": 0.0,
       "step": 9
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 1024.0,
+      "completion_length": 512.0,
       "epoch": 0.005714285714285714,
-      "kl": 0.0,
-      "learning_rate": 0.0,
-      "loss": 0.0,
-      "reward": -0.7860384881496429,
-      "reward_std": 0.29622524976730347,
-      "rewards/cosine_scaled_reward": -0.39301924407482147,
+      "kl": 0.0041675567626953125,
+      "learning_rate": 7.039090644965509e-07,
+      "loss": 0.00016715750098228455,
+      "reward": -0.860070988535881,
+      "reward_std": 0.17052607703953981,
+      "rewards/cosine_scaled_reward": -0.4300354793667793,
       "rewards/format_reward": 0.0,
       "step": 10
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 1024.0,
+      "completion_length": 477.125,
       "epoch": 0.006285714285714286,
-      "kl": 0.0,
-      "learning_rate": 1.0000000000000002e-06,
-      "loss": -1.2665987014770508e-06,
-      "reward": -0.9470862597227097,
-      "reward_std": 0.025707244174554944,
-      "rewards/cosine_scaled_reward": -0.47354312986135483,
+      "kl": 0.003407001495361328,
+      "learning_rate": 6.281416799501187e-07,
+      "loss": 0.00013599544763565063,
+      "reward": -0.8231181800365448,
+      "reward_std": 0.1259385095909238,
+      "rewards/cosine_scaled_reward": -0.4115590825676918,
       "rewards/format_reward": 0.0,
       "step": 11
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 1024.0,
+      "completion_length": 512.0,
       "epoch": 0.006857142857142857,
-      "kl": 0.0,
-      "learning_rate": 1.0000000000000002e-06,
-      "loss": -3.2782554626464844e-07,
-      "reward": -0.9222527593374252,
-      "reward_std": 0.11853919085115194,
-      "rewards/cosine_scaled_reward": -0.4611263796687126,
+      "kl": 0.0012102127075195312,
+      "learning_rate": 5.5e-07,
+      "loss": 5.13419508934021e-05,
+      "reward": -0.9447444081306458,
+      "reward_std": 0.08061030774842948,
+      "rewards/cosine_scaled_reward": -0.4723722040653229,
       "rewards/format_reward": 0.0,
       "step": 12
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 1024.0,
+      "completion_length": 487.9375,
       "epoch": 0.0074285714285714285,
-      "kl": -2.9742717742919922e-05,
-      "learning_rate": 2.0000000000000003e-06,
-      "loss": -9.834766387939453e-07,
-      "reward": -0.8048228025436401,
-      "reward_std": 0.2713719364255667,
-      "rewards/cosine_scaled_reward": -0.40241140127182007,
+      "kl": 0.00119781494140625,
+      "learning_rate": 4.7185832004988133e-07,
+      "loss": 0.07869705557823181,
+      "reward": -0.8934571295976639,
+      "reward_std": 0.15774485282599926,
+      "rewards/cosine_scaled_reward": -0.44672856479883194,
       "rewards/format_reward": 0.0,
       "step": 13
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 1024.0,
+      "completion_length": 466.625,
       "epoch": 0.008,
-      "kl": -1.9848346710205078e-05,
-      "learning_rate": 3e-06,
-      "loss": -9.98377799987793e-07,
-      "reward": -0.8982286602258682,
-      "reward_std": 0.15008432138711214,
-      "rewards/cosine_scaled_reward": -0.4491143301129341,
+      "kl": 0.004004955291748047,
+      "learning_rate": 3.9609093550344907e-07,
+      "loss": 0.027497582137584686,
+      "reward": -0.7841525673866272,
+      "reward_std": 0.2220854666084051,
+      "rewards/cosine_scaled_reward": -0.3920762911438942,
       "rewards/format_reward": 0.0,
       "step": 14
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 1024.0,
+      "completion_length": 497.5,
       "epoch": 0.008571428571428572,
-      "kl": -3.173947334289551e-05,
-      "learning_rate": 4.000000000000001e-06,
-      "loss": -1.2367963790893555e-06,
-      "reward": -0.8698235750198364,
-      "reward_std": 0.1454296549782157,
-      "rewards/cosine_scaled_reward": -0.4349117949604988,
+      "kl": 0.012208938598632812,
+      "learning_rate": 3.250000000000001e-07,
+      "loss": 0.0004888176918029785,
+      "reward": -0.7216114401817322,
+      "reward_std": 0.3453192347660661,
+      "rewards/cosine_scaled_reward": -0.3608057275414467,
       "rewards/format_reward": 0.0,
       "step": 15
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 1024.0,
+      "completion_length": 499.0,
       "epoch": 0.009142857142857144,
-      "kl": -2.2411346435546875e-05,
-      "learning_rate": 5e-06,
-      "loss": -1.166015863418579e-06,
-      "reward": -0.8962945342063904,
-      "reward_std": 0.11415091808885336,
-      "rewards/cosine_scaled_reward": -0.4481472671031952,
+      "kl": 0.003875732421875,
+      "learning_rate": 2.6074557564105724e-07,
+      "loss": 0.00015526264905929565,
+      "reward": -0.8265384286642075,
+      "reward_std": 0.25778803089633584,
+      "rewards/cosine_scaled_reward": -0.4132692217826843,
       "rewards/format_reward": 0.0,
       "step": 16
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 1024.0,
+      "completion_length": 501.5,
       "epoch": 0.009714285714285713,
-      "kl": -9.268522262573242e-06,
-      "learning_rate": 6e-06,
-      "loss": -4.172325134277344e-07,
-      "reward": -0.8075007796287537,
-      "reward_std": 0.2773375315591693,
-      "rewards/cosine_scaled_reward": -0.40375038236379623,
+      "kl": 0.004252910614013672,
+      "learning_rate": 2.0528000059645995e-07,
+      "loss": 0.02472507953643799,
+      "reward": -0.8701069504022598,
+      "reward_std": 0.18937412789091468,
+      "rewards/cosine_scaled_reward": -0.4350534752011299,
       "rewards/format_reward": 0.0,
       "step": 17
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 1024.0,
+      "completion_length": 506.625,
       "epoch": 0.010285714285714285,
-      "kl": -1.519918441772461e-06,
-      "learning_rate": 7e-06,
-      "loss": 1.0356307029724121e-06,
-      "reward": -0.8843557834625244,
-      "reward_std": 0.1459937175968662,
-      "rewards/cosine_scaled_reward": -0.4421778917312622,
+      "kl": 0.005939483642578125,
+      "learning_rate": 1.6028856829700258e-07,
+      "loss": 0.00023746490478515625,
+      "reward": -0.8200783580541611,
+      "reward_std": 0.26157089229673147,
+      "rewards/cosine_scaled_reward": -0.41003918647766113,
       "rewards/format_reward": 0.0,
       "step": 18
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 1024.0,
+      "completion_length": 446.25,
       "epoch": 0.010857142857142857,
-      "kl": -1.1920928955078125e-07,
-      "learning_rate": 8.000000000000001e-06,
-      "loss": -1.4901161193847656e-08,
-      "reward": -0.9325414896011353,
-      "reward_std": 0.061190704349428415,
-      "rewards/cosine_scaled_reward": -0.46627073734998703,
+      "kl": 0.002822399139404297,
+      "learning_rate": 1.2713832064634125e-07,
+      "loss": 0.016675502061843872,
+      "reward": -0.8233697563409805,
+      "reward_std": 0.20013628248125315,
+      "rewards/cosine_scaled_reward": -0.41168487817049026,
       "rewards/format_reward": 0.0,
       "step": 19
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 1024.0,
+      "completion_length": 496.125,
       "epoch": 0.011428571428571429,
-      "kl": -7.152557373046875e-07,
-      "learning_rate": 9e-06,
-      "loss": 7.450580596923828e-08,
-      "reward": -0.8927065879106522,
-      "reward_std": 0.1468478236347437,
-      "rewards/cosine_scaled_reward": -0.4463533014059067,
+      "kl": 0.00678253173828125,
+      "learning_rate": 1.068365111445064e-07,
+      "loss": 0.00027292221784591675,
+      "reward": -0.8044158518314362,
+      "reward_std": 0.2423506089253351,
+      "rewards/cosine_scaled_reward": -0.4022079259157181,
       "rewards/format_reward": 0.0,
       "step": 20
     },
     {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.012,
-      "kl": -4.470348358154297e-07,
-      "learning_rate": 1e-05,
-      "loss": -3.241002559661865e-07,
-      "reward": -0.861713707447052,
-      "reward_std": 0.15973031288012862,
-      "rewards/cosine_scaled_reward": -0.430856853723526,
-      "rewards/format_reward": 0.0,
-      "step": 21
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.012571428571428572,
-      "kl": 2.980232238769531e-07,
-      "learning_rate": 9.997258721585931e-06,
-      "loss": 0.0,
-      "reward": -0.8567932546138763,
-      "reward_std": 0.23303062841296196,
-      "rewards/cosine_scaled_reward": -0.42839662730693817,
-      "rewards/format_reward": 0.0,
-      "step": 22
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.013142857142857144,
-      "kl": 6.258487701416016e-07,
-      "learning_rate": 9.98903822616921e-06,
-      "loss": -1.9371509552001953e-07,
-      "reward": -0.9055128693580627,
-      "reward_std": 0.13258774112910032,
-      "rewards/cosine_scaled_reward": -0.45275644212961197,
-      "rewards/format_reward": 0.0,
-      "step": 23
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.013714285714285714,
-      "kl": 2.682209014892578e-07,
-      "learning_rate": 9.97534852915723e-06,
-      "loss": 4.3958425521850586e-07,
-      "reward": -0.9318011105060577,
-      "reward_std": 0.08933348534628749,
-      "rewards/cosine_scaled_reward": -0.46590057015419006,
-      "rewards/format_reward": 0.0,
-      "step": 24
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.014285714285714285,
-      "kl": 3.337860107421875e-06,
-      "learning_rate": 9.956206309337067e-06,
-      "loss": -8.940696716308594e-08,
-      "reward": -0.9025110006332397,
-      "reward_std": 0.15539621422067285,
-      "rewards/cosine_scaled_reward": -0.45125550776720047,
-      "rewards/format_reward": 0.0,
-      "step": 25
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.014857142857142857,
-      "kl": 2.086162567138672e-06,
-      "learning_rate": 9.931634888554937e-06,
-      "loss": -2.8312206268310547e-07,
-      "reward": -0.9135829955339432,
-      "reward_std": 0.08904895093291998,
-      "rewards/cosine_scaled_reward": -0.456791490316391,
-      "rewards/format_reward": 0.0,
-      "step": 26
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.015428571428571429,
-      "kl": 2.8312206268310547e-06,
-      "learning_rate": 9.901664203302126e-06,
-      "loss": -2.60770320892334e-06,
-      "reward": -0.950261116027832,
-      "reward_std": 0.06268075766274706,
-      "rewards/cosine_scaled_reward": -0.4751305654644966,
-      "rewards/format_reward": 0.0,
-      "step": 27
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.016,
-      "kl": 2.8014183044433594e-06,
-      "learning_rate": 9.866330768241984e-06,
-      "loss": 5.960464477539063e-08,
-      "reward": -0.8735224008560181,
-      "reward_std": 0.16378713678568602,
-      "rewards/cosine_scaled_reward": -0.4367612153291702,
-      "rewards/format_reward": 0.0,
-      "step": 28
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.01657142857142857,
-      "kl": 1.0013580322265625e-05,
-      "learning_rate": 9.825677631722436e-06,
-      "loss": 4.842877388000488e-07,
-      "reward": -0.856491282582283,
-      "reward_std": 0.1944181639701128,
-      "rewards/cosine_scaled_reward": -0.4282456487417221,
-      "rewards/format_reward": 0.0,
-      "step": 29
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.017142857142857144,
-      "kl": 6.258487701416016e-06,
-      "learning_rate": 9.779754323328192e-06,
-      "loss": 1.0058283805847168e-06,
-      "reward": -0.9048939347267151,
-      "reward_std": 0.1466370872221887,
-      "rewards/cosine_scaled_reward": -0.45244697481393814,
-      "rewards/format_reward": 0.0,
-      "step": 30
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.017714285714285714,
-      "kl": 1.4841556549072266e-05,
-      "learning_rate": 9.728616793536588e-06,
-      "loss": 7.301568984985352e-07,
-      "reward": -0.9575549513101578,
-      "reward_std": 0.04152237856760621,
-      "rewards/cosine_scaled_reward": -0.4787774831056595,
-      "rewards/format_reward": 0.0,
-      "step": 31
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.018285714285714287,
-      "kl": 1.5348196029663086e-05,
-      "learning_rate": 9.672327345550544e-06,
-      "loss": 7.525086402893066e-07,
-      "reward": -0.9280864149332047,
-      "reward_std": 0.05714223568793386,
-      "rewards/cosine_scaled_reward": -0.4640432074666023,
-      "rewards/format_reward": 0.0,
-      "step": 32
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.018857142857142857,
-      "kl": 2.396106719970703e-05,
-      "learning_rate": 9.610954559391704e-06,
-      "loss": 6.705522537231445e-07,
-      "reward": -0.8499859273433685,
-      "reward_std": 0.13816553819924593,
-      "rewards/cosine_scaled_reward": -0.42499294877052307,
-      "rewards/format_reward": 0.0,
-      "step": 33
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.019428571428571427,
-      "kl": 2.98917293548584e-05,
-      "learning_rate": 9.544573208346252e-06,
-      "loss": 3.4868717193603516e-06,
-      "reward": -0.9111520648002625,
-      "reward_std": 0.06401598325464875,
-      "rewards/cosine_scaled_reward": -0.4555760398507118,
-      "rewards/format_reward": 0.0,
-      "step": 34
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.02,
-      "kl": 5.048513412475586e-05,
-      "learning_rate": 9.473264167865172e-06,
-      "loss": 1.996755599975586e-06,
-      "reward": -0.9233099222183228,
-      "reward_std": 0.07124835508875549,
-      "rewards/cosine_scaled_reward": -0.4616549611091614,
-      "rewards/format_reward": 0.0,
-      "step": 35
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.02057142857142857,
-      "kl": 7.835030555725098e-05,
-      "learning_rate": 9.397114317029975e-06,
-      "loss": 3.5315752029418945e-06,
-      "reward": -0.8740164637565613,
-      "reward_std": 0.1880413582548499,
-      "rewards/cosine_scaled_reward": -0.43700823187828064,
-      "rewards/format_reward": 0.0,
-      "step": 36
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 960.4375,
-      "epoch": 0.021142857142857144,
-      "kl": 0.0002009570598602295,
-      "learning_rate": 9.316216432703918e-06,
-      "loss": 0.1341422200202942,
-      "reward": -0.8138948976993561,
-      "reward_std": 0.2712679710239172,
-      "rewards/cosine_scaled_reward": -0.40694746375083923,
-      "rewards/format_reward": 0.0,
-      "step": 37
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.021714285714285714,
-      "kl": 0.00011423230171203613,
-      "learning_rate": 9.230669076497688e-06,
-      "loss": 4.366040229797363e-06,
-      "reward": -0.9074572026729584,
-      "reward_std": 0.140884583350271,
-      "rewards/cosine_scaled_reward": -0.4537286013364792,
-      "rewards/format_reward": 0.0,
-      "step": 38
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.022285714285714287,
-      "kl": 7.608532905578613e-05,
-      "learning_rate": 9.140576474687263e-06,
-      "loss": 2.9802322387695312e-06,
-      "reward": -0.9341968446969986,
-      "reward_std": 0.08859914634376764,
-      "rewards/cosine_scaled_reward": -0.4670984223484993,
-      "rewards/format_reward": 0.0,
-      "step": 39
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.022857142857142857,
-      "kl": 8.574128150939941e-05,
-      "learning_rate": 9.046048391230248e-06,
-      "loss": 3.2633543014526367e-06,
-      "reward": -0.9477669596672058,
-      "reward_std": 0.03630512161180377,
-      "rewards/cosine_scaled_reward": -0.4738834798336029,
-      "rewards/format_reward": 0.0,
-      "step": 40
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.023428571428571427,
-      "kl": 9.319186210632324e-05,
-      "learning_rate": 8.947199994035402e-06,
-      "loss": 3.5390257835388184e-06,
-      "reward": -0.9622567743062973,
-      "reward_std": 0.03198406333103776,
-      "rewards/cosine_scaled_reward": -0.48112839460372925,
-      "rewards/format_reward": 0.0,
-      "step": 41
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.024,
-      "kl": 0.00011175870895385742,
-      "learning_rate": 8.844151714648274e-06,
-      "loss": 4.059635102748871e-06,
-      "reward": -0.9137657284736633,
-      "reward_std": 0.09395218873396516,
-      "rewards/cosine_scaled_reward": -0.45688286423683167,
-      "rewards/format_reward": 0.0,
-      "step": 42
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.02457142857142857,
-      "kl": 9.465217590332031e-05,
-      "learning_rate": 8.737029101523931e-06,
-      "loss": 3.972090780735016e-06,
-      "reward": -0.8712927997112274,
-      "reward_std": 0.17926698923110962,
-      "rewards/cosine_scaled_reward": -0.4356464073061943,
-      "rewards/format_reward": 0.0,
-      "step": 43
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.025142857142857144,
-      "kl": 0.00018846988677978516,
-      "learning_rate": 8.625962667065488e-06,
-      "loss": 7.338821887969971e-06,
-      "reward": -0.8325228244066238,
-      "reward_std": 0.22310001868754625,
-      "rewards/cosine_scaled_reward": -0.4162614122033119,
-      "rewards/format_reward": 0.0,
-      "step": 44
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.025714285714285714,
-      "kl": 0.00025010108947753906,
-      "learning_rate": 8.511087728614863e-06,
-      "loss": 9.98377799987793e-06,
-      "reward": -0.8755671381950378,
-      "reward_std": 0.1843216335400939,
-      "rewards/cosine_scaled_reward": -0.4377835765480995,
-      "rewards/format_reward": 0.0,
-      "step": 45
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.026285714285714287,
-      "kl": 0.00020682811737060547,
-      "learning_rate": 8.392544243589428e-06,
-      "loss": 7.063150405883789e-06,
-      "reward": -0.9294558465480804,
-      "reward_std": 0.08181617665104568,
-      "rewards/cosine_scaled_reward": -0.4647279307246208,
-      "rewards/format_reward": 0.0,
-      "step": 46
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.026857142857142857,
-      "kl": 0.00023487210273742676,
-      "learning_rate": 8.270476638965463e-06,
-      "loss": 9.715557098388672e-06,
-      "reward": -0.9637707620859146,
-      "reward_std": 0.04111184738576412,
-      "rewards/cosine_scaled_reward": -0.4818853959441185,
-      "rewards/format_reward": 0.0,
-      "step": 47
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.027428571428571427,
-      "kl": 0.00025856494903564453,
-      "learning_rate": 8.14503363531613e-06,
-      "loss": 1.0699033737182617e-05,
-      "reward": -0.9429489076137543,
-      "reward_std": 0.05583410756662488,
-      "rewards/cosine_scaled_reward": -0.47147445380687714,
-      "rewards/format_reward": 0.0,
-      "step": 48
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.028,
-      "kl": 0.00023242831230163574,
-      "learning_rate": 8.016368065618361e-06,
-      "loss": 9.488314390182495e-06,
-      "reward": -0.9678349643945694,
-      "reward_std": 0.031033652368932962,
-      "rewards/cosine_scaled_reward": -0.4839174821972847,
-      "rewards/format_reward": 0.0,
-      "step": 49
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.02857142857142857,
-      "kl": 0.0002980530261993408,
-      "learning_rate": 7.884636689049423e-06,
-      "loss": 1.1995434761047363e-05,
-      "reward": -0.92691770195961,
-      "reward_std": 0.06057529430836439,
-      "rewards/cosine_scaled_reward": -0.4634588584303856,
-      "rewards/format_reward": 0.0,
-      "step": 50
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.029142857142857144,
-      "kl": 0.0002821683883666992,
-      "learning_rate": 7.75e-06,
-      "loss": 1.093745231628418e-05,
-      "reward": -0.9709955900907516,
-      "reward_std": 0.023253681138157845,
-      "rewards/cosine_scaled_reward": -0.48549777269363403,
-      "rewards/format_reward": 0.0,
-      "step": 51
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.029714285714285714,
-      "kl": 0.0003021657466888428,
-      "learning_rate": 7.612622032536508e-06,
-      "loss": 1.2755393981933594e-05,
-      "reward": -0.932096004486084,
-      "reward_std": 0.0814181575551629,
-      "rewards/cosine_scaled_reward": -0.4660480171442032,
-      "rewards/format_reward": 0.0,
-      "step": 52
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.030285714285714287,
-      "kl": 0.0003120899200439453,
-      "learning_rate": 7.472670160550849e-06,
-      "loss": 1.1809170246124268e-05,
-      "reward": -0.8598470240831375,
-      "reward_std": 0.23317514825612307,
-      "rewards/cosine_scaled_reward": -0.42992351949214935,
-      "rewards/format_reward": 0.0,
-      "step": 53
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.030857142857142857,
-      "kl": 0.00031960010528564453,
-      "learning_rate": 7.330314893841102e-06,
-      "loss": 1.2435019016265869e-05,
-      "reward": -0.8917691111564636,
-      "reward_std": 0.16836319211870432,
-      "rewards/cosine_scaled_reward": -0.4458845555782318,
-      "rewards/format_reward": 0.0,
-      "step": 54
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.03142857142857143,
-      "kl": 0.0003180503845214844,
-      "learning_rate": 7.185729670371605e-06,
-      "loss": 1.2367963790893555e-05,
-      "reward": -0.9393985569477081,
-      "reward_std": 0.09125666646286845,
-      "rewards/cosine_scaled_reward": -0.46969927847385406,
-      "rewards/format_reward": 0.0,
-      "step": 55
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.032,
-      "kl": 0.00032332539558410645,
-      "learning_rate": 7.0390906449655104e-06,
-      "loss": 1.3027340173721313e-05,
-      "reward": -0.9556453377008438,
-      "reward_std": 0.03563447529450059,
-      "rewards/cosine_scaled_reward": -0.4778226688504219,
-      "rewards/format_reward": 0.0,
-      "step": 56
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.03257142857142857,
-      "kl": 0.00029200315475463867,
-      "learning_rate": 6.890576474687264e-06,
-      "loss": 1.173466444015503e-05,
-      "reward": -0.9533008933067322,
-      "reward_std": 0.047348865773528814,
-      "rewards/cosine_scaled_reward": -0.4766504615545273,
-      "rewards/format_reward": 0.0,
-      "step": 57
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.03314285714285714,
-      "kl": 0.00028020143508911133,
-      "learning_rate": 6.740368101176496e-06,
-      "loss": 1.0944902896881104e-05,
-      "reward": -0.8606822788715363,
-      "reward_std": 0.19956759549677372,
-      "rewards/cosine_scaled_reward": -0.4303411394357681,
-      "rewards/format_reward": 0.0,
-      "step": 58
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.03371428571428572,
-      "kl": 0.00034242868423461914,
-      "learning_rate": 6.588648530198505e-06,
-      "loss": 1.4327466487884521e-05,
-      "reward": -0.9435078203678131,
-      "reward_std": 0.028082083677873015,
-      "rewards/cosine_scaled_reward": -0.47175391018390656,
-      "rewards/format_reward": 0.0,
-      "step": 59
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.03428571428571429,
-      "kl": 0.0004596710205078125,
-      "learning_rate": 6.4356026086799176e-06,
-      "loss": 1.862645149230957e-05,
-      "reward": -0.8699476420879364,
-      "reward_std": 0.14623272977769375,
-      "rewards/cosine_scaled_reward": -0.4349738284945488,
-      "rewards/format_reward": 0.0,
-      "step": 60
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.03485714285714286,
-      "kl": 0.00041598081588745117,
-      "learning_rate": 6.281416799501188e-06,
-      "loss": 1.6495585441589355e-05,
-      "reward": -0.8786782920360565,
-      "reward_std": 0.14873503288254142,
-      "rewards/cosine_scaled_reward": -0.43933914601802826,
-      "rewards/format_reward": 0.0,
-      "step": 61
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.03542857142857143,
-      "kl": 0.0003428459167480469,
-      "learning_rate": 6.126278954320295e-06,
-      "loss": 1.4573335647583008e-05,
-      "reward": -0.9531020373106003,
-      "reward_std": 0.04140742728486657,
-      "rewards/cosine_scaled_reward": -0.47655102610588074,
-      "rewards/format_reward": 0.0,
-      "step": 62
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.036,
-      "kl": 0.00034630298614501953,
-      "learning_rate": 5.970378084704441e-06,
-      "loss": 1.3574957847595215e-05,
-      "reward": -0.9625549018383026,
-      "reward_std": 0.0323515310883522,
-      "rewards/cosine_scaled_reward": -0.4812774509191513,
-      "rewards/format_reward": 0.0,
-      "step": 63
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.036571428571428574,
-      "kl": 0.000436633825302124,
-      "learning_rate": 5.813904131848565e-06,
-      "loss": 1.7490237951278687e-05,
-      "reward": -0.9638274163007736,
-      "reward_std": 0.03202465921640396,
-      "rewards/cosine_scaled_reward": -0.4819137006998062,
-      "rewards/format_reward": 0.0,
-      "step": 64
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.037142857142857144,
-      "kl": 0.00037598609924316406,
-      "learning_rate": 5.657047735161256e-06,
-      "loss": 1.4990568161010742e-05,
-      "reward": -0.8853072375059128,
-      "reward_std": 0.11870932951569557,
-      "rewards/cosine_scaled_reward": -0.4426536113023758,
-      "rewards/format_reward": 0.0,
-      "step": 65
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.037714285714285714,
-      "kl": 0.000420987606048584,
-      "learning_rate": 5.500000000000001e-06,
-      "loss": 1.6763806343078613e-05,
-      "reward": -0.9631430506706238,
-      "reward_std": 0.027665999252349138,
-      "rewards/cosine_scaled_reward": -0.4815715327858925,
-      "rewards/format_reward": 0.0,
-      "step": 66
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.038285714285714284,
-      "kl": 0.0005689859390258789,
-      "learning_rate": 5.342952264838748e-06,
-      "loss": 2.3305416107177734e-05,
-      "reward": -0.9261669814586639,
-      "reward_std": 0.09819867718033493,
-      "rewards/cosine_scaled_reward": -0.46308349817991257,
-      "rewards/format_reward": 0.0,
-      "step": 67
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.038857142857142854,
-      "kl": 0.0006139874458312988,
-      "learning_rate": 5.186095868151436e-06,
-      "loss": 2.4430453777313232e-05,
-      "reward": -0.9322563856840134,
-      "reward_std": 0.06839151354506612,
-      "rewards/cosine_scaled_reward": -0.4661281928420067,
-      "rewards/format_reward": 0.0,
-      "step": 68
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.03942857142857143,
-      "kl": 0.0004551410675048828,
-      "learning_rate": 5.02962191529556e-06,
-      "loss": 1.753866672515869e-05,
-      "reward": -0.947336733341217,
-      "reward_std": 0.06602911371737719,
-      "rewards/cosine_scaled_reward": -0.4736683666706085,
-      "rewards/format_reward": 0.0,
-      "step": 69
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.04,
-      "kl": 0.0004698038101196289,
-      "learning_rate": 4.873721045679707e-06,
-      "loss": 1.9043684005737305e-05,
-      "reward": -0.881416529417038,
-      "reward_std": 0.15695088542997837,
-      "rewards/cosine_scaled_reward": -0.440708264708519,
-      "rewards/format_reward": 0.0,
-      "step": 70
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.04057142857142857,
-      "kl": 0.0005980134010314941,
-      "learning_rate": 4.718583200498814e-06,
-      "loss": 2.4437904357910156e-05,
-      "reward": -0.9552861750125885,
-      "reward_std": 0.04716450162231922,
-      "rewards/cosine_scaled_reward": -0.47764309495687485,
-      "rewards/format_reward": 0.0,
-      "step": 71
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.04114285714285714,
-      "kl": 0.00044411420822143555,
-      "learning_rate": 4.564397391320085e-06,
-      "loss": 1.781061291694641e-05,
-      "reward": -0.9175944626331329,
-      "reward_std": 0.07855925057083368,
-      "rewards/cosine_scaled_reward": -0.45879723131656647,
-      "rewards/format_reward": 0.0,
-      "step": 72
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.04171428571428572,
-      "kl": 0.0005142688751220703,
-      "learning_rate": 4.4113514698014955e-06,
-      "loss": 2.104230225086212e-05,
-      "reward": -0.9591091871261597,
-      "reward_std": 0.03032594360411167,
-      "rewards/cosine_scaled_reward": -0.47955460846424103,
-      "rewards/format_reward": 0.0,
-      "step": 73
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.04228571428571429,
-      "kl": 0.0004570484161376953,
-      "learning_rate": 4.259631898823504e-06,
-      "loss": 1.8224120140075684e-05,
-      "reward": -0.9418470114469528,
-      "reward_std": 0.09771440364420414,
-      "rewards/cosine_scaled_reward": -0.470923513174057,
-      "rewards/format_reward": 0.0,
-      "step": 74
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.04285714285714286,
-      "kl": 0.00042492151260375977,
-      "learning_rate": 4.109423525312738e-06,
-      "loss": 1.697242259979248e-05,
-      "reward": -0.9031769037246704,
-      "reward_std": 0.11431426065973938,
-      "rewards/cosine_scaled_reward": -0.4515884444117546,
-      "rewards/format_reward": 0.0,
-      "step": 75
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.04342857142857143,
-      "kl": 0.0003921389579772949,
-      "learning_rate": 3.960909355034491e-06,
-      "loss": 1.5683472156524658e-05,
-      "reward": -0.8710711896419525,
-      "reward_std": 0.16496195830404758,
-      "rewards/cosine_scaled_reward": -0.43553559482097626,
-      "rewards/format_reward": 0.0,
-      "step": 76
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.044,
-      "kl": 0.0005483031272888184,
-      "learning_rate": 3.8142703296283954e-06,
-      "loss": 2.2258609533309937e-05,
-      "reward": -0.8363375961780548,
-      "reward_std": 0.23763815127313137,
-      "rewards/cosine_scaled_reward": -0.418168805539608,
-      "rewards/format_reward": 0.0,
-      "step": 77
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.044571428571428574,
-      "kl": 0.0005096793174743652,
-      "learning_rate": 3.6696851061589e-06,
-      "loss": 2.06679105758667e-05,
-      "reward": -0.912532389163971,
-      "reward_std": 0.1374040930531919,
-      "rewards/cosine_scaled_reward": -0.45626621693372726,
-      "rewards/format_reward": 0.0,
-      "step": 78
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.045142857142857144,
-      "kl": 0.0007692575454711914,
-      "learning_rate": 3.527329839449152e-06,
-      "loss": 3.053247928619385e-05,
-      "reward": -0.9567261040210724,
-      "reward_std": 0.03342599933966994,
-      "rewards/cosine_scaled_reward": -0.478363037109375,
-      "rewards/format_reward": 0.0,
-      "step": 79
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.045714285714285714,
-      "kl": 0.0008780956268310547,
-      "learning_rate": 3.3873779674634932e-06,
-      "loss": 3.489106893539429e-05,
-      "reward": -0.8690381050109863,
-      "reward_std": 0.166263896971941,
-      "rewards/cosine_scaled_reward": -0.43451904505491257,
-      "rewards/format_reward": 0.0,
-      "step": 80
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.046285714285714284,
-      "kl": 0.000619053840637207,
-      "learning_rate": 3.2500000000000015e-06,
-      "loss": 2.2970139980316162e-05,
-      "reward": -0.9703131467103958,
-      "reward_std": 0.018532322952523828,
-      "rewards/cosine_scaled_reward": -0.4851565733551979,
-      "rewards/format_reward": 0.0,
-      "step": 81
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.046857142857142854,
-      "kl": 0.0005697011947631836,
-      "learning_rate": 3.115363310950579e-06,
-      "loss": 2.2605061531066895e-05,
-      "reward": -0.8376921266317368,
-      "reward_std": 0.21680933889001608,
-      "rewards/cosine_scaled_reward": -0.4188460633158684,
-      "rewards/format_reward": 0.0,
-      "step": 82
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.04742857142857143,
-      "kl": 0.0006507635116577148,
-      "learning_rate": 2.98363193438164e-06,
-      "loss": 2.580881118774414e-05,
-      "reward": -0.9194770008325577,
-      "reward_std": 0.07644858444109559,
-      "rewards/cosine_scaled_reward": -0.45973849296569824,
-      "rewards/format_reward": 0.0,
-      "step": 83
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.048,
-      "kl": 0.0005425214767456055,
-      "learning_rate": 2.854966364683872e-06,
-      "loss": 2.232193946838379e-05,
-      "reward": -0.921579897403717,
-      "reward_std": 0.07768060895614326,
-      "rewards/cosine_scaled_reward": -0.4607899561524391,
-      "rewards/format_reward": 0.0,
-      "step": 84
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.04857142857142857,
-      "kl": 0.0005598664283752441,
-      "learning_rate": 2.7295233610345384e-06,
-      "loss": 2.230703830718994e-05,
-      "reward": -0.8130292594432831,
-      "reward_std": 0.21775285061448812,
-      "rewards/cosine_scaled_reward": -0.40651462972164154,
-      "rewards/format_reward": 0.0,
-      "step": 85
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.04914285714285714,
-      "kl": 0.0005010366439819336,
-      "learning_rate": 2.607455756410573e-06,
-      "loss": 2.0213425159454346e-05,
-      "reward": -0.911712571978569,
-      "reward_std": 0.09181239921599627,
-      "rewards/cosine_scaled_reward": -0.4558562785387039,
-      "rewards/format_reward": 0.0,
-      "step": 86
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.04971428571428571,
-      "kl": 0.0005140304565429688,
-      "learning_rate": 2.4889122713851397e-06,
-      "loss": 2.09808349609375e-05,
-      "reward": -0.9401500076055527,
-      "reward_std": 0.04285714589059353,
-      "rewards/cosine_scaled_reward": -0.47007501125335693,
-      "rewards/format_reward": 0.0,
-      "step": 87
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.05028571428571429,
-      "kl": 0.0005880594253540039,
-      "learning_rate": 2.374037332934512e-06,
-      "loss": 2.3573637008666992e-05,
-      "reward": -0.8337626904249191,
-      "reward_std": 0.24908871483057737,
-      "rewards/cosine_scaled_reward": -0.41688134521245956,
-      "rewards/format_reward": 0.0,
-      "step": 88
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.05085714285714286,
-      "kl": 0.0005828142166137695,
-      "learning_rate": 2.262970898476071e-06,
-      "loss": 2.3312866687774658e-05,
-      "reward": -0.9419961720705032,
-      "reward_std": 0.06526243314146996,
-      "rewards/cosine_scaled_reward": -0.4709980934858322,
-      "rewards/format_reward": 0.0,
-      "step": 89
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.05142857142857143,
-      "kl": 0.0005592107772827148,
-      "learning_rate": 2.1558482853517257e-06,
-      "loss": 2.16066837310791e-05,
-      "reward": -0.9420932680368423,
-      "reward_std": 0.05442978721112013,
-      "rewards/cosine_scaled_reward": -0.47104664146900177,
-      "rewards/format_reward": 0.0,
-      "step": 90
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.052,
-      "kl": 0.0005729198455810547,
-      "learning_rate": 2.0528000059646e-06,
-      "loss": 2.3186206817626953e-05,
-      "reward": -0.8448977470397949,
-      "reward_std": 0.17764843348413706,
-      "rewards/cosine_scaled_reward": -0.42244888097047806,
-      "rewards/format_reward": 0.0,
-      "step": 91
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.052571428571428575,
-      "kl": 0.0006086826324462891,
-      "learning_rate": 1.953951608769752e-06,
-      "loss": 2.4143606424331665e-05,
-      "reward": -0.8687557131052017,
-      "reward_std": 0.13139949878677726,
-      "rewards/cosine_scaled_reward": -0.43437784910202026,
-      "rewards/format_reward": 0.0,
-      "step": 92
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.053142857142857144,
-      "kl": 0.0004508495330810547,
-      "learning_rate": 1.8594235253127373e-06,
-      "loss": 1.7457641661167145e-05,
-      "reward": -0.9452957063913345,
-      "reward_std": 0.044932478107512,
-      "rewards/cosine_scaled_reward": -0.4726478382945061,
-      "rewards/format_reward": 0.0,
-      "step": 93
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.053714285714285714,
-      "kl": 0.00058746337890625,
-      "learning_rate": 1.769330923502313e-06,
-      "loss": 2.3424625396728516e-05,
-      "reward": -0.9732565432786942,
-      "reward_std": 0.023667596746236086,
-      "rewards/cosine_scaled_reward": -0.48662828654050827,
-      "rewards/format_reward": 0.0,
-      "step": 94
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.054285714285714284,
-      "kl": 0.0006615519523620605,
-      "learning_rate": 1.6837835672960834e-06,
-      "loss": 2.664327621459961e-05,
-      "reward": -0.8294684588909149,
-      "reward_std": 0.21260959655046463,
-      "rewards/cosine_scaled_reward": -0.41473422944545746,
-      "rewards/format_reward": 0.0,
-      "step": 95
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.054857142857142854,
-      "kl": 0.0005936622619628906,
-      "learning_rate": 1.602885682970026e-06,
-      "loss": 2.310052514076233e-05,
-      "reward": -0.8817574381828308,
-      "reward_std": 0.14791762363165617,
-      "rewards/cosine_scaled_reward": -0.4408787190914154,
-      "rewards/format_reward": 0.0,
-      "step": 96
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.05542857142857143,
-      "kl": 0.0004170536994934082,
-      "learning_rate": 1.526735832134829e-06,
-      "loss": 1.5437602996826172e-05,
-      "reward": -0.9217518121004105,
-      "reward_std": 0.05548063712194562,
-      "rewards/cosine_scaled_reward": -0.46087589859962463,
-      "rewards/format_reward": 0.0,
-      "step": 97
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.056,
-      "kl": 0.0005135536193847656,
-      "learning_rate": 1.4554267916537495e-06,
-      "loss": 2.017989754676819e-05,
-      "reward": -0.9414444863796234,
-      "reward_std": 0.057337059173732996,
-      "rewards/cosine_scaled_reward": -0.4707222431898117,
-      "rewards/format_reward": 0.0,
-      "step": 98
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.05657142857142857,
-      "kl": 0.0004296302795410156,
-      "learning_rate": 1.389045440608296e-06,
-      "loss": 1.6748905181884766e-05,
-      "reward": -0.8684801608324051,
-      "reward_std": 0.18490357510745525,
-      "rewards/cosine_scaled_reward": -0.43424008786678314,
-      "rewards/format_reward": 0.0,
-      "step": 99
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 1024.0,
-      "epoch": 0.05714285714285714,
-      "kl": 0.0005539655685424805,
-      "learning_rate": 1.3276726544494572e-06,
-      "loss": 2.2009015083312988e-05,
-      "reward": -0.8967786431312561,
-      "reward_std": 0.12791539868339896,
-      "rewards/cosine_scaled_reward": -0.44838932156562805,
-      "rewards/format_reward": 0.0,
-      "step": 100
-    },
-    {
-      "epoch": 0.05714285714285714,
-      "step": 100,
+      "epoch": 0.011428571428571429,
+      "step": 20,
       "total_flos": 0.0,
-      "train_loss": 0.0013520326372236013,
-      "train_runtime": 36067.9715,
-      "train_samples_per_second": 0.044,
-      "train_steps_per_second": 0.003
+      "train_loss": 0.010430806130170823,
+      "train_runtime": 9458.0577,
+      "train_samples_per_second": 0.034,
+      "train_steps_per_second": 0.002
     }
   ],
   "logging_steps": 1,
-  "max_steps": 100,
+  "max_steps": 20,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 50,

training_metrics.txt

@@ -1,6 +1,6 @@
-total_size_before (MB): 3424.75
-total_size_after (MB): 3407.14
-total_time (seconds): 36073.19
-ram_peak (MB): 5853.69
-ram_consump (MB): 3485.15
-disk_storage (MB): 333.13
+total_size_before (MB): 1455.72
+total_size_after (MB): 1445.40
+total_time (seconds): 9466.33
+ram_peak (MB): 3499.90
+ram_consump (MB): 1492.71
+disk_storage (MB): 616.56