Model save

README.md

@@ -1,11 +1,9 @@
 ---
-base_model: Qwen/Qwen3.5-0.8B
-datasets: knoveleng/open-rs
+base_model: deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B
 library_name: transformers
 model_name: OpenRS-GRPO
 tags:
 - generated_from_trainer
-- open-r1
 - trl
 - grpo
 licence: license
@@ -13,7 +11,7 @@ licence: license
 
 # Model Card for OpenRS-GRPO
 
-This model is a fine-tuned version of [Qwen/Qwen3.5-0.8B](https://huggingface.co/Qwen/Qwen3.5-0.8B) on the [knoveleng/open-rs](https://huggingface.co/datasets/knoveleng/open-rs) dataset.
+This model is a fine-tuned version of [deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B).
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 4.023313522338867e-07,
-    "train_runtime": 537.7965,
+    "train_loss": 0.0013520326372236013,
+    "train_runtime": 36067.9715,
     "train_samples": 7000,
-    "train_samples_per_second": 0.015,
-    "train_steps_per_second": 0.002
+    "train_samples_per_second": 0.044,
+    "train_steps_per_second": 0.003
 }

step_metrics.csv

@@ -1,3 +1,102 @@
 step,epoch,loss,learning_rate,grad_norm,rewards/format_reward,rewards/cosine_scaled_reward,reward,reward_std,gpu_mem_alloc_mb,gpu_mem_peak_mb,step_time_sec
-1,0.0006,4.023313522338867e-07,0.0,,0.0,-0.40046167373657227,-0.8009233474731445,0.2397190211340785,1549.0,3351.1,533.87
-1,0.0006,,,,,,,,1549.0,3351.1,537.79
+1,0.0006,3.3527612686157227e-08,0.0,,0.0,-0.45186813920736313,-0.9037362784147263,0.11632500402629375,3617.5,6086.5,356.01
+2,0.0011,-2.9802322387695312e-08,0.0,,0.0,-0.4544064328074455,-0.9088128805160522,0.134795643389225,3617.5,6091.3,357.14
+3,0.0017,-2.384185791015625e-07,0.0,,0.0,-0.44806819409132004,-0.8961364179849625,0.1266492954455316,3617.5,6092.8,356.69
+4,0.0023,3.203749656677246e-07,0.0,,0.0,-0.46924517303705215,-0.9384903311729431,0.08165389159694314,3617.5,6095.2,358.13
+5,0.0029,2.980232238769531e-07,0.0,,0.0,-0.4361194893717766,-0.8722389936447144,0.1861576708033681,3617.5,6095.2,355.88
+6,0.0034,3.2782554626464844e-07,0.0,,0.0,-0.4838990569114685,-0.9677980989217758,0.03695695102214813,3617.5,6102.0,359.97
+7,0.004,-6.48200511932373e-07,0.0,,0.0,-0.46913372725248337,-0.9382674694061279,0.04741490981541574,3617.5,6102.0,366.73
+8,0.0046,-1.9371509552001953e-07,0.0,,0.0,-0.4689921736717224,-0.9379843473434448,0.08557501714676619,3617.5,6102.0,359.73
+9,0.0051,6.258487701416016e-07,0.0,,0.0,-0.47829224169254303,-0.9565844535827637,0.04161944845691323,3617.5,6102.0,356.95
+10,0.0057,0.0,0.0,,0.0,-0.39301924407482147,-0.7860384881496429,0.29622524976730347,3654.4,6102.0,356.9
+11,0.0063,-1.2665987014770508e-06,1.0000000000000002e-06,,0.0,-0.47354312986135483,-0.9470862597227097,0.025707244174554944,3654.4,6129.7,357.65
+12,0.0069,-3.2782554626464844e-07,1.0000000000000002e-06,,0.0,-0.4611263796687126,-0.9222527593374252,0.11853919085115194,3654.4,6132.1,359.0
+13,0.0074,-9.834766387939453e-07,2.0000000000000003e-06,,0.0,-0.40241140127182007,-0.8048228025436401,0.2713719364255667,3654.4,6132.1,359.35
+14,0.008,-9.98377799987793e-07,3e-06,,0.0,-0.4491143301129341,-0.8982286602258682,0.15008432138711214,3654.4,6132.1,359.42
+15,0.0086,-1.2367963790893555e-06,4.000000000000001e-06,,0.0,-0.4349117949604988,-0.8698235750198364,0.1454296549782157,3654.4,6132.1,358.18
+16,0.0091,-1.166015863418579e-06,5e-06,,0.0,-0.4481472671031952,-0.8962945342063904,0.11415091808885336,3654.4,6136.0,359.05
+17,0.0097,-4.172325134277344e-07,6e-06,,0.0,-0.40375038236379623,-0.8075007796287537,0.2773375315591693,3654.4,6136.0,359.81
+18,0.0103,1.0356307029724121e-06,7e-06,,0.0,-0.4421778917312622,-0.8843557834625244,0.1459937175968662,3654.4,6136.0,361.88
+19,0.0109,-1.4901161193847656e-08,8.000000000000001e-06,,0.0,-0.46627073734998703,-0.9325414896011353,0.061190704349428415,3654.4,6136.0,361.95
+20,0.0114,7.450580596923828e-08,9e-06,,0.0,-0.4463533014059067,-0.8927065879106522,0.1468478236347437,3654.4,6136.0,361.89
+21,0.012,-3.241002559661865e-07,1e-05,,0.0,-0.430856853723526,-0.861713707447052,0.15973031288012862,3654.4,6136.0,360.43
+22,0.0126,0.0,9.997258721585931e-06,,0.0,-0.42839662730693817,-0.8567932546138763,0.23303062841296196,3654.4,6136.0,359.97
+23,0.0131,-1.9371509552001953e-07,9.98903822616921e-06,,0.0,-0.45275644212961197,-0.9055128693580627,0.13258774112910032,3654.4,6136.0,360.57
+24,0.0137,4.3958425521850586e-07,9.97534852915723e-06,,0.0,-0.46590057015419006,-0.9318011105060577,0.08933348534628749,3654.4,6136.0,362.48
+25,0.0143,-8.940696716308594e-08,9.956206309337067e-06,,0.0,-0.45125550776720047,-0.9025110006332397,0.15539621422067285,3654.4,6136.0,364.1
+26,0.0149,-2.8312206268310547e-07,9.931634888554937e-06,,0.0,-0.456791490316391,-0.9135829955339432,0.08904895093291998,3654.4,6136.0,365.75
+27,0.0154,-2.60770320892334e-06,9.901664203302126e-06,,0.0,-0.4751305654644966,-0.950261116027832,0.06268075766274706,3654.4,6136.0,370.26
+28,0.016,5.960464477539063e-08,9.866330768241984e-06,,0.0,-0.4367612153291702,-0.8735224008560181,0.16378713678568602,3654.4,6138.0,366.74
+29,0.0166,4.842877388000488e-07,9.825677631722436e-06,,0.0,-0.4282456487417221,-0.856491282582283,0.1944181639701128,3654.5,6138.0,364.94
+30,0.0171,1.0058283805847168e-06,9.779754323328192e-06,,0.0,-0.45244697481393814,-0.9048939347267151,0.1466370872221887,3654.4,6138.0,359.72
+31,0.0177,7.301568984985352e-07,9.728616793536588e-06,,0.0,-0.4787774831056595,-0.9575549513101578,0.04152237856760621,3654.4,6138.0,359.74
+32,0.0183,7.525086402893066e-07,9.672327345550544e-06,,0.0,-0.4640432074666023,-0.9280864149332047,0.05714223568793386,3654.4,6138.0,359.53
+33,0.0189,6.705522537231445e-07,9.610954559391704e-06,,0.0,-0.42499294877052307,-0.8499859273433685,0.13816553819924593,3654.5,6138.0,362.56
+34,0.0194,3.4868717193603516e-06,9.544573208346252e-06,,0.0,-0.4555760398507118,-0.9111520648002625,0.06401598325464875,3654.4,6138.0,362.18
+35,0.02,1.996755599975586e-06,9.473264167865172e-06,,0.0,-0.4616549611091614,-0.9233099222183228,0.07124835508875549,3654.4,6138.0,360.07
+36,0.0206,3.5315752029418945e-06,9.397114317029975e-06,,0.0,-0.43700823187828064,-0.8740164637565613,0.1880413582548499,3654.4,6138.0,358.61
+37,0.0211,0.1341422200202942,9.316216432703918e-06,,0.0,-0.40694746375083923,-0.8138948976993561,0.2712679710239172,3654.4,6138.0,357.92
+38,0.0217,4.366040229797363e-06,9.230669076497688e-06,,0.0,-0.4537286013364792,-0.9074572026729584,0.140884583350271,3654.5,6138.0,361.01
+39,0.0223,2.9802322387695312e-06,9.140576474687263e-06,,0.0,-0.4670984223484993,-0.9341968446969986,0.08859914634376764,3654.4,6138.0,358.53
+40,0.0229,3.2633543014526367e-06,9.046048391230248e-06,,0.0,-0.4738834798336029,-0.9477669596672058,0.03630512161180377,3654.4,6138.0,356.31
+41,0.0234,3.5390257835388184e-06,8.947199994035402e-06,,0.0,-0.48112839460372925,-0.9622567743062973,0.03198406333103776,3654.4,6138.0,356.46
+42,0.024,4.059635102748871e-06,8.844151714648274e-06,,0.0,-0.45688286423683167,-0.9137657284736633,0.09395218873396516,3654.4,6138.0,358.44
+43,0.0246,3.972090780735016e-06,8.737029101523931e-06,,0.0,-0.4356464073061943,-0.8712927997112274,0.17926698923110962,3654.4,6138.0,362.88
+44,0.0251,7.338821887969971e-06,8.625962667065488e-06,,0.0,-0.4162614122033119,-0.8325228244066238,0.22310001868754625,3654.4,6138.0,358.73
+45,0.0257,9.98377799987793e-06,8.511087728614863e-06,,0.0,-0.4377835765480995,-0.8755671381950378,0.1843216335400939,3654.4,6138.0,359.1
+46,0.0263,7.063150405883789e-06,8.392544243589428e-06,,0.0,-0.4647279307246208,-0.9294558465480804,0.08181617665104568,3654.4,6138.0,364.32
+47,0.0269,9.715557098388672e-06,8.270476638965463e-06,,0.0,-0.4818853959441185,-0.9637707620859146,0.04111184738576412,3654.4,6138.0,366.26
+48,0.0274,1.0699033737182617e-05,8.14503363531613e-06,,0.0,-0.47147445380687714,-0.9429489076137543,0.05583410756662488,3654.4,6138.0,367.58
+49,0.028,9.488314390182495e-06,8.016368065618361e-06,,0.0,-0.4839174821972847,-0.9678349643945694,0.031033652368932962,3654.4,6138.0,367.03
+50,0.0286,1.1995434761047363e-05,7.884636689049423e-06,,0.0,-0.4634588584303856,-0.92691770195961,0.06057529430836439,3654.4,6138.0,365.37
+51,0.0291,1.093745231628418e-05,7.75e-06,,0.0,-0.48549777269363403,-0.9709955900907516,0.023253681138157845,3654.5,6138.0,357.96
+52,0.0297,1.2755393981933594e-05,7.612622032536508e-06,,0.0,-0.4660480171442032,-0.932096004486084,0.0814181575551629,3654.5,6138.0,357.69
+53,0.0303,1.1809170246124268e-05,7.472670160550849e-06,,0.0,-0.42992351949214935,-0.8598470240831375,0.23317514825612307,3654.5,6138.0,360.51
+54,0.0309,1.2435019016265869e-05,7.330314893841102e-06,,0.0,-0.4458845555782318,-0.8917691111564636,0.16836319211870432,3654.5,6138.0,359.99
+55,0.0314,1.2367963790893555e-05,7.185729670371605e-06,,0.0,-0.46969927847385406,-0.9393985569477081,0.09125666646286845,3654.5,6138.0,357.92
+56,0.032,1.3027340173721313e-05,7.0390906449655104e-06,,0.0,-0.4778226688504219,-0.9556453377008438,0.03563447529450059,3654.4,6138.0,359.54
+57,0.0326,1.173466444015503e-05,6.890576474687264e-06,,0.0,-0.4766504615545273,-0.9533008933067322,0.047348865773528814,3654.5,6138.0,360.89
+58,0.0331,1.0944902896881104e-05,6.740368101176496e-06,,0.0,-0.4303411394357681,-0.8606822788715363,0.19956759549677372,3654.5,6138.0,357.96
+59,0.0337,1.4327466487884521e-05,6.588648530198505e-06,,0.0,-0.47175391018390656,-0.9435078203678131,0.028082083677873015,3654.4,6138.0,358.32
+60,0.0343,1.862645149230957e-05,6.4356026086799176e-06,,0.0,-0.4349738284945488,-0.8699476420879364,0.14623272977769375,3654.5,6138.0,358.57
+61,0.0349,1.6495585441589355e-05,6.281416799501188e-06,,0.0,-0.43933914601802826,-0.8786782920360565,0.14873503288254142,3654.5,6138.0,359.81
+62,0.0354,1.4573335647583008e-05,6.126278954320295e-06,,0.0,-0.47655102610588074,-0.9531020373106003,0.04140742728486657,3654.5,6138.0,363.55
+63,0.036,1.3574957847595215e-05,5.970378084704441e-06,,0.0,-0.4812774509191513,-0.9625549018383026,0.0323515310883522,3654.4,6138.0,361.67
+64,0.0366,1.7490237951278687e-05,5.813904131848565e-06,,0.0,-0.4819137006998062,-0.9638274163007736,0.03202465921640396,3654.5,6138.0,360.66
+65,0.0371,1.4990568161010742e-05,5.657047735161256e-06,,0.0,-0.4426536113023758,-0.8853072375059128,0.11870932951569557,3654.5,6138.0,360.47
+66,0.0377,1.6763806343078613e-05,5.500000000000001e-06,,0.0,-0.4815715327858925,-0.9631430506706238,0.027665999252349138,3654.4,6138.0,364.95
+67,0.0383,2.3305416107177734e-05,5.342952264838748e-06,,0.0,-0.46308349817991257,-0.9261669814586639,0.09819867718033493,3654.4,6138.0,369.54
+68,0.0389,2.4430453777313232e-05,5.186095868151436e-06,,0.0,-0.4661281928420067,-0.9322563856840134,0.06839151354506612,3654.4,6138.0,369.64
+69,0.0394,1.753866672515869e-05,5.02962191529556e-06,,0.0,-0.4736683666706085,-0.947336733341217,0.06602911371737719,3654.5,6138.0,365.41
+70,0.04,1.9043684005737305e-05,4.873721045679707e-06,,0.0,-0.440708264708519,-0.881416529417038,0.15695088542997837,3654.4,6138.0,364.8
+71,0.0406,2.4437904357910156e-05,4.718583200498814e-06,,0.0,-0.47764309495687485,-0.9552861750125885,0.04716450162231922,3654.4,6138.0,362.55
+72,0.0411,1.781061291694641e-05,4.564397391320085e-06,,0.0,-0.45879723131656647,-0.9175944626331329,0.07855925057083368,3654.4,6138.0,359.86
+73,0.0417,2.104230225086212e-05,4.4113514698014955e-06,,0.0,-0.47955460846424103,-0.9591091871261597,0.03032594360411167,3654.4,6138.0,354.97
+74,0.0423,1.8224120140075684e-05,4.259631898823504e-06,,0.0,-0.470923513174057,-0.9418470114469528,0.09771440364420414,3654.4,6138.0,359.71
+75,0.0429,1.697242259979248e-05,4.109423525312738e-06,,0.0,-0.4515884444117546,-0.9031769037246704,0.11431426065973938,3654.4,6138.0,359.51
+76,0.0434,1.5683472156524658e-05,3.960909355034491e-06,,0.0,-0.43553559482097626,-0.8710711896419525,0.16496195830404758,3654.4,6138.0,357.76
+77,0.044,2.2258609533309937e-05,3.8142703296283954e-06,,0.0,-0.418168805539608,-0.8363375961780548,0.23763815127313137,3654.4,6138.0,359.55
+78,0.0446,2.06679105758667e-05,3.6696851061589e-06,,0.0,-0.45626621693372726,-0.912532389163971,0.1374040930531919,3654.5,6138.0,359.56
+79,0.0451,3.053247928619385e-05,3.527329839449152e-06,,0.0,-0.478363037109375,-0.9567261040210724,0.03342599933966994,3654.4,6138.0,359.31
+80,0.0457,3.489106893539429e-05,3.3873779674634932e-06,,0.0,-0.43451904505491257,-0.8690381050109863,0.166263896971941,3654.5,6138.0,358.11
+81,0.0463,2.2970139980316162e-05,3.2500000000000015e-06,,0.0,-0.4851565733551979,-0.9703131467103958,0.018532322952523828,3654.4,6138.0,359.52
+82,0.0469,2.2605061531066895e-05,3.115363310950579e-06,,0.0,-0.4188460633158684,-0.8376921266317368,0.21680933889001608,3654.5,6138.0,360.43
+83,0.0474,2.580881118774414e-05,2.98363193438164e-06,,0.0,-0.45973849296569824,-0.9194770008325577,0.07644858444109559,3654.4,6138.0,357.84
+84,0.048,2.232193946838379e-05,2.854966364683872e-06,,0.0,-0.4607899561524391,-0.921579897403717,0.07768060895614326,3654.4,6138.0,364.25
+85,0.0486,2.230703830718994e-05,2.7295233610345384e-06,,0.0,-0.40651462972164154,-0.8130292594432831,0.21775285061448812,3654.4,6138.0,361.8
+86,0.0491,2.0213425159454346e-05,2.607455756410573e-06,,0.0,-0.4558562785387039,-0.911712571978569,0.09181239921599627,3654.5,6138.0,363.18
+87,0.0497,2.09808349609375e-05,2.4889122713851397e-06,,0.0,-0.47007501125335693,-0.9401500076055527,0.04285714589059353,3654.5,6138.0,368.87
+88,0.0503,2.3573637008666992e-05,2.374037332934512e-06,,0.0,-0.41688134521245956,-0.8337626904249191,0.24908871483057737,3654.4,6138.0,360.15
+89,0.0509,2.3312866687774658e-05,2.262970898476071e-06,,0.0,-0.4709980934858322,-0.9419961720705032,0.06526243314146996,3654.5,6138.0,355.62
+90,0.0514,2.16066837310791e-05,2.1558482853517257e-06,,0.0,-0.47104664146900177,-0.9420932680368423,0.05442978721112013,3654.4,6138.0,359.25
+91,0.052,2.3186206817626953e-05,2.0528000059646e-06,,0.0,-0.42244888097047806,-0.8448977470397949,0.17764843348413706,3654.4,6138.0,357.58
+92,0.0526,2.4143606424331665e-05,1.953951608769752e-06,,0.0,-0.43437784910202026,-0.8687557131052017,0.13139949878677726,3654.5,6138.0,360.98
+93,0.0531,1.7457641661167145e-05,1.8594235253127373e-06,,0.0,-0.4726478382945061,-0.9452957063913345,0.044932478107512,3654.4,6138.0,359.65
+94,0.0537,2.3424625396728516e-05,1.769330923502313e-06,,0.0,-0.48662828654050827,-0.9732565432786942,0.023667596746236086,3654.4,6138.0,357.33
+95,0.0543,2.664327621459961e-05,1.6837835672960834e-06,,0.0,-0.41473422944545746,-0.8294684588909149,0.21260959655046463,3654.4,6138.0,360.81
+96,0.0549,2.310052514076233e-05,1.602885682970026e-06,,0.0,-0.4408787190914154,-0.8817574381828308,0.14791762363165617,3654.4,6138.0,359.17
+97,0.0554,1.5437602996826172e-05,1.526735832134829e-06,,0.0,-0.46087589859962463,-0.9217518121004105,0.05548063712194562,3654.5,6138.0,357.98
+98,0.056,2.017989754676819e-05,1.4554267916537495e-06,,0.0,-0.4707222431898117,-0.9414444863796234,0.057337059173732996,3654.4,6138.0,359.07
+99,0.0566,1.6748905181884766e-05,1.389045440608296e-06,,0.0,-0.43424008786678314,-0.8684801608324051,0.18490357510745525,3654.4,6138.0,357.48
+100,0.0571,2.2009015083312988e-05,1.3276726544494572e-06,,0.0,-0.44838932156562805,-0.8967786431312561,0.12791539868339896,3654.4,6138.0,358.57
+100,0.0571,,,,,,,,3654.4,6138.0,362.18

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 4.023313522338867e-07,
-    "train_runtime": 537.7965,
+    "train_loss": 0.0013520326372236013,
+    "train_runtime": 36067.9715,
     "train_samples": 7000,
-    "train_samples_per_second": 0.015,
-    "train_steps_per_second": 0.002
+    "train_samples_per_second": 0.044,
+    "train_steps_per_second": 0.003
 }

trainer_state.json

@@ -2,38 +2,1325 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.0005714285714285715,
+  "epoch": 0.05714285714285714,
   "eval_steps": 500,
-  "global_step": 1,
+  "global_step": 100,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "clip_ratio": 0.0,
-      "completion_length": 445.25,
+      "completion_length": 1024.0,
       "epoch": 0.0005714285714285715,
       "kl": 0.0,
       "learning_rate": 0.0,
-      "loss": 4.023313522338867e-07,
-      "reward": -0.8009233474731445,
-      "reward_std": 0.2397190211340785,
-      "rewards/cosine_scaled_reward": -0.40046167373657227,
+      "loss": 3.3527612686157227e-08,
+      "reward": -0.9037362784147263,
+      "reward_std": 0.11632500402629375,
+      "rewards/cosine_scaled_reward": -0.45186813920736313,
       "rewards/format_reward": 0.0,
       "step": 1
     },
     {
-      "epoch": 0.0005714285714285715,
-      "step": 1,
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.001142857142857143,
+      "kl": 0.0,
+      "learning_rate": 0.0,
+      "loss": -2.9802322387695312e-08,
+      "reward": -0.9088128805160522,
+      "reward_std": 0.134795643389225,
+      "rewards/cosine_scaled_reward": -0.4544064328074455,
+      "rewards/format_reward": 0.0,
+      "step": 2
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.0017142857142857142,
+      "kl": 0.0,
+      "learning_rate": 0.0,
+      "loss": -2.384185791015625e-07,
+      "reward": -0.8961364179849625,
+      "reward_std": 0.1266492954455316,
+      "rewards/cosine_scaled_reward": -0.44806819409132004,
+      "rewards/format_reward": 0.0,
+      "step": 3
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.002285714285714286,
+      "kl": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.203749656677246e-07,
+      "reward": -0.9384903311729431,
+      "reward_std": 0.08165389159694314,
+      "rewards/cosine_scaled_reward": -0.46924517303705215,
+      "rewards/format_reward": 0.0,
+      "step": 4
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.002857142857142857,
+      "kl": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.980232238769531e-07,
+      "reward": -0.8722389936447144,
+      "reward_std": 0.1861576708033681,
+      "rewards/cosine_scaled_reward": -0.4361194893717766,
+      "rewards/format_reward": 0.0,
+      "step": 5
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.0034285714285714284,
+      "kl": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.2782554626464844e-07,
+      "reward": -0.9677980989217758,
+      "reward_std": 0.03695695102214813,
+      "rewards/cosine_scaled_reward": -0.4838990569114685,
+      "rewards/format_reward": 0.0,
+      "step": 6
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.004,
+      "kl": 0.0,
+      "learning_rate": 0.0,
+      "loss": -6.48200511932373e-07,
+      "reward": -0.9382674694061279,
+      "reward_std": 0.04741490981541574,
+      "rewards/cosine_scaled_reward": -0.46913372725248337,
+      "rewards/format_reward": 0.0,
+      "step": 7
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.004571428571428572,
+      "kl": 0.0,
+      "learning_rate": 0.0,
+      "loss": -1.9371509552001953e-07,
+      "reward": -0.9379843473434448,
+      "reward_std": 0.08557501714676619,
+      "rewards/cosine_scaled_reward": -0.4689921736717224,
+      "rewards/format_reward": 0.0,
+      "step": 8
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.005142857142857143,
+      "kl": 0.0,
+      "learning_rate": 0.0,
+      "loss": 6.258487701416016e-07,
+      "reward": -0.9565844535827637,
+      "reward_std": 0.04161944845691323,
+      "rewards/cosine_scaled_reward": -0.47829224169254303,
+      "rewards/format_reward": 0.0,
+      "step": 9
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.005714285714285714,
+      "kl": 0.0,
+      "learning_rate": 0.0,
+      "loss": 0.0,
+      "reward": -0.7860384881496429,
+      "reward_std": 0.29622524976730347,
+      "rewards/cosine_scaled_reward": -0.39301924407482147,
+      "rewards/format_reward": 0.0,
+      "step": 10
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.006285714285714286,
+      "kl": 0.0,
+      "learning_rate": 1.0000000000000002e-06,
+      "loss": -1.2665987014770508e-06,
+      "reward": -0.9470862597227097,
+      "reward_std": 0.025707244174554944,
+      "rewards/cosine_scaled_reward": -0.47354312986135483,
+      "rewards/format_reward": 0.0,
+      "step": 11
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.006857142857142857,
+      "kl": 0.0,
+      "learning_rate": 1.0000000000000002e-06,
+      "loss": -3.2782554626464844e-07,
+      "reward": -0.9222527593374252,
+      "reward_std": 0.11853919085115194,
+      "rewards/cosine_scaled_reward": -0.4611263796687126,
+      "rewards/format_reward": 0.0,
+      "step": 12
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.0074285714285714285,
+      "kl": -2.9742717742919922e-05,
+      "learning_rate": 2.0000000000000003e-06,
+      "loss": -9.834766387939453e-07,
+      "reward": -0.8048228025436401,
+      "reward_std": 0.2713719364255667,
+      "rewards/cosine_scaled_reward": -0.40241140127182007,
+      "rewards/format_reward": 0.0,
+      "step": 13
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.008,
+      "kl": -1.9848346710205078e-05,
+      "learning_rate": 3e-06,
+      "loss": -9.98377799987793e-07,
+      "reward": -0.8982286602258682,
+      "reward_std": 0.15008432138711214,
+      "rewards/cosine_scaled_reward": -0.4491143301129341,
+      "rewards/format_reward": 0.0,
+      "step": 14
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.008571428571428572,
+      "kl": -3.173947334289551e-05,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": -1.2367963790893555e-06,
+      "reward": -0.8698235750198364,
+      "reward_std": 0.1454296549782157,
+      "rewards/cosine_scaled_reward": -0.4349117949604988,
+      "rewards/format_reward": 0.0,
+      "step": 15
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.009142857142857144,
+      "kl": -2.2411346435546875e-05,
+      "learning_rate": 5e-06,
+      "loss": -1.166015863418579e-06,
+      "reward": -0.8962945342063904,
+      "reward_std": 0.11415091808885336,
+      "rewards/cosine_scaled_reward": -0.4481472671031952,
+      "rewards/format_reward": 0.0,
+      "step": 16
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.009714285714285713,
+      "kl": -9.268522262573242e-06,
+      "learning_rate": 6e-06,
+      "loss": -4.172325134277344e-07,
+      "reward": -0.8075007796287537,
+      "reward_std": 0.2773375315591693,
+      "rewards/cosine_scaled_reward": -0.40375038236379623,
+      "rewards/format_reward": 0.0,
+      "step": 17
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.010285714285714285,
+      "kl": -1.519918441772461e-06,
+      "learning_rate": 7e-06,
+      "loss": 1.0356307029724121e-06,
+      "reward": -0.8843557834625244,
+      "reward_std": 0.1459937175968662,
+      "rewards/cosine_scaled_reward": -0.4421778917312622,
+      "rewards/format_reward": 0.0,
+      "step": 18
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.010857142857142857,
+      "kl": -1.1920928955078125e-07,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": -1.4901161193847656e-08,
+      "reward": -0.9325414896011353,
+      "reward_std": 0.061190704349428415,
+      "rewards/cosine_scaled_reward": -0.46627073734998703,
+      "rewards/format_reward": 0.0,
+      "step": 19
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.011428571428571429,
+      "kl": -7.152557373046875e-07,
+      "learning_rate": 9e-06,
+      "loss": 7.450580596923828e-08,
+      "reward": -0.8927065879106522,
+      "reward_std": 0.1468478236347437,
+      "rewards/cosine_scaled_reward": -0.4463533014059067,
+      "rewards/format_reward": 0.0,
+      "step": 20
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.012,
+      "kl": -4.470348358154297e-07,
+      "learning_rate": 1e-05,
+      "loss": -3.241002559661865e-07,
+      "reward": -0.861713707447052,
+      "reward_std": 0.15973031288012862,
+      "rewards/cosine_scaled_reward": -0.430856853723526,
+      "rewards/format_reward": 0.0,
+      "step": 21
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.012571428571428572,
+      "kl": 2.980232238769531e-07,
+      "learning_rate": 9.997258721585931e-06,
+      "loss": 0.0,
+      "reward": -0.8567932546138763,
+      "reward_std": 0.23303062841296196,
+      "rewards/cosine_scaled_reward": -0.42839662730693817,
+      "rewards/format_reward": 0.0,
+      "step": 22
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.013142857142857144,
+      "kl": 6.258487701416016e-07,
+      "learning_rate": 9.98903822616921e-06,
+      "loss": -1.9371509552001953e-07,
+      "reward": -0.9055128693580627,
+      "reward_std": 0.13258774112910032,
+      "rewards/cosine_scaled_reward": -0.45275644212961197,
+      "rewards/format_reward": 0.0,
+      "step": 23
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.013714285714285714,
+      "kl": 2.682209014892578e-07,
+      "learning_rate": 9.97534852915723e-06,
+      "loss": 4.3958425521850586e-07,
+      "reward": -0.9318011105060577,
+      "reward_std": 0.08933348534628749,
+      "rewards/cosine_scaled_reward": -0.46590057015419006,
+      "rewards/format_reward": 0.0,
+      "step": 24
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.014285714285714285,
+      "kl": 3.337860107421875e-06,
+      "learning_rate": 9.956206309337067e-06,
+      "loss": -8.940696716308594e-08,
+      "reward": -0.9025110006332397,
+      "reward_std": 0.15539621422067285,
+      "rewards/cosine_scaled_reward": -0.45125550776720047,
+      "rewards/format_reward": 0.0,
+      "step": 25
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.014857142857142857,
+      "kl": 2.086162567138672e-06,
+      "learning_rate": 9.931634888554937e-06,
+      "loss": -2.8312206268310547e-07,
+      "reward": -0.9135829955339432,
+      "reward_std": 0.08904895093291998,
+      "rewards/cosine_scaled_reward": -0.456791490316391,
+      "rewards/format_reward": 0.0,
+      "step": 26
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.015428571428571429,
+      "kl": 2.8312206268310547e-06,
+      "learning_rate": 9.901664203302126e-06,
+      "loss": -2.60770320892334e-06,
+      "reward": -0.950261116027832,
+      "reward_std": 0.06268075766274706,
+      "rewards/cosine_scaled_reward": -0.4751305654644966,
+      "rewards/format_reward": 0.0,
+      "step": 27
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.016,
+      "kl": 2.8014183044433594e-06,
+      "learning_rate": 9.866330768241984e-06,
+      "loss": 5.960464477539063e-08,
+      "reward": -0.8735224008560181,
+      "reward_std": 0.16378713678568602,
+      "rewards/cosine_scaled_reward": -0.4367612153291702,
+      "rewards/format_reward": 0.0,
+      "step": 28
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.01657142857142857,
+      "kl": 1.0013580322265625e-05,
+      "learning_rate": 9.825677631722436e-06,
+      "loss": 4.842877388000488e-07,
+      "reward": -0.856491282582283,
+      "reward_std": 0.1944181639701128,
+      "rewards/cosine_scaled_reward": -0.4282456487417221,
+      "rewards/format_reward": 0.0,
+      "step": 29
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.017142857142857144,
+      "kl": 6.258487701416016e-06,
+      "learning_rate": 9.779754323328192e-06,
+      "loss": 1.0058283805847168e-06,
+      "reward": -0.9048939347267151,
+      "reward_std": 0.1466370872221887,
+      "rewards/cosine_scaled_reward": -0.45244697481393814,
+      "rewards/format_reward": 0.0,
+      "step": 30
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.017714285714285714,
+      "kl": 1.4841556549072266e-05,
+      "learning_rate": 9.728616793536588e-06,
+      "loss": 7.301568984985352e-07,
+      "reward": -0.9575549513101578,
+      "reward_std": 0.04152237856760621,
+      "rewards/cosine_scaled_reward": -0.4787774831056595,
+      "rewards/format_reward": 0.0,
+      "step": 31
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.018285714285714287,
+      "kl": 1.5348196029663086e-05,
+      "learning_rate": 9.672327345550544e-06,
+      "loss": 7.525086402893066e-07,
+      "reward": -0.9280864149332047,
+      "reward_std": 0.05714223568793386,
+      "rewards/cosine_scaled_reward": -0.4640432074666023,
+      "rewards/format_reward": 0.0,
+      "step": 32
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.018857142857142857,
+      "kl": 2.396106719970703e-05,
+      "learning_rate": 9.610954559391704e-06,
+      "loss": 6.705522537231445e-07,
+      "reward": -0.8499859273433685,
+      "reward_std": 0.13816553819924593,
+      "rewards/cosine_scaled_reward": -0.42499294877052307,
+      "rewards/format_reward": 0.0,
+      "step": 33
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.019428571428571427,
+      "kl": 2.98917293548584e-05,
+      "learning_rate": 9.544573208346252e-06,
+      "loss": 3.4868717193603516e-06,
+      "reward": -0.9111520648002625,
+      "reward_std": 0.06401598325464875,
+      "rewards/cosine_scaled_reward": -0.4555760398507118,
+      "rewards/format_reward": 0.0,
+      "step": 34
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.02,
+      "kl": 5.048513412475586e-05,
+      "learning_rate": 9.473264167865172e-06,
+      "loss": 1.996755599975586e-06,
+      "reward": -0.9233099222183228,
+      "reward_std": 0.07124835508875549,
+      "rewards/cosine_scaled_reward": -0.4616549611091614,
+      "rewards/format_reward": 0.0,
+      "step": 35
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.02057142857142857,
+      "kl": 7.835030555725098e-05,
+      "learning_rate": 9.397114317029975e-06,
+      "loss": 3.5315752029418945e-06,
+      "reward": -0.8740164637565613,
+      "reward_std": 0.1880413582548499,
+      "rewards/cosine_scaled_reward": -0.43700823187828064,
+      "rewards/format_reward": 0.0,
+      "step": 36
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 960.4375,
+      "epoch": 0.021142857142857144,
+      "kl": 0.0002009570598602295,
+      "learning_rate": 9.316216432703918e-06,
+      "loss": 0.1341422200202942,
+      "reward": -0.8138948976993561,
+      "reward_std": 0.2712679710239172,
+      "rewards/cosine_scaled_reward": -0.40694746375083923,
+      "rewards/format_reward": 0.0,
+      "step": 37
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.021714285714285714,
+      "kl": 0.00011423230171203613,
+      "learning_rate": 9.230669076497688e-06,
+      "loss": 4.366040229797363e-06,
+      "reward": -0.9074572026729584,
+      "reward_std": 0.140884583350271,
+      "rewards/cosine_scaled_reward": -0.4537286013364792,
+      "rewards/format_reward": 0.0,
+      "step": 38
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.022285714285714287,
+      "kl": 7.608532905578613e-05,
+      "learning_rate": 9.140576474687263e-06,
+      "loss": 2.9802322387695312e-06,
+      "reward": -0.9341968446969986,
+      "reward_std": 0.08859914634376764,
+      "rewards/cosine_scaled_reward": -0.4670984223484993,
+      "rewards/format_reward": 0.0,
+      "step": 39
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.022857142857142857,
+      "kl": 8.574128150939941e-05,
+      "learning_rate": 9.046048391230248e-06,
+      "loss": 3.2633543014526367e-06,
+      "reward": -0.9477669596672058,
+      "reward_std": 0.03630512161180377,
+      "rewards/cosine_scaled_reward": -0.4738834798336029,
+      "rewards/format_reward": 0.0,
+      "step": 40
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.023428571428571427,
+      "kl": 9.319186210632324e-05,
+      "learning_rate": 8.947199994035402e-06,
+      "loss": 3.5390257835388184e-06,
+      "reward": -0.9622567743062973,
+      "reward_std": 0.03198406333103776,
+      "rewards/cosine_scaled_reward": -0.48112839460372925,
+      "rewards/format_reward": 0.0,
+      "step": 41
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.024,
+      "kl": 0.00011175870895385742,
+      "learning_rate": 8.844151714648274e-06,
+      "loss": 4.059635102748871e-06,
+      "reward": -0.9137657284736633,
+      "reward_std": 0.09395218873396516,
+      "rewards/cosine_scaled_reward": -0.45688286423683167,
+      "rewards/format_reward": 0.0,
+      "step": 42
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.02457142857142857,
+      "kl": 9.465217590332031e-05,
+      "learning_rate": 8.737029101523931e-06,
+      "loss": 3.972090780735016e-06,
+      "reward": -0.8712927997112274,
+      "reward_std": 0.17926698923110962,
+      "rewards/cosine_scaled_reward": -0.4356464073061943,
+      "rewards/format_reward": 0.0,
+      "step": 43
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.025142857142857144,
+      "kl": 0.00018846988677978516,
+      "learning_rate": 8.625962667065488e-06,
+      "loss": 7.338821887969971e-06,
+      "reward": -0.8325228244066238,
+      "reward_std": 0.22310001868754625,
+      "rewards/cosine_scaled_reward": -0.4162614122033119,
+      "rewards/format_reward": 0.0,
+      "step": 44
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.025714285714285714,
+      "kl": 0.00025010108947753906,
+      "learning_rate": 8.511087728614863e-06,
+      "loss": 9.98377799987793e-06,
+      "reward": -0.8755671381950378,
+      "reward_std": 0.1843216335400939,
+      "rewards/cosine_scaled_reward": -0.4377835765480995,
+      "rewards/format_reward": 0.0,
+      "step": 45
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.026285714285714287,
+      "kl": 0.00020682811737060547,
+      "learning_rate": 8.392544243589428e-06,
+      "loss": 7.063150405883789e-06,
+      "reward": -0.9294558465480804,
+      "reward_std": 0.08181617665104568,
+      "rewards/cosine_scaled_reward": -0.4647279307246208,
+      "rewards/format_reward": 0.0,
+      "step": 46
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.026857142857142857,
+      "kl": 0.00023487210273742676,
+      "learning_rate": 8.270476638965463e-06,
+      "loss": 9.715557098388672e-06,
+      "reward": -0.9637707620859146,
+      "reward_std": 0.04111184738576412,
+      "rewards/cosine_scaled_reward": -0.4818853959441185,
+      "rewards/format_reward": 0.0,
+      "step": 47
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.027428571428571427,
+      "kl": 0.00025856494903564453,
+      "learning_rate": 8.14503363531613e-06,
+      "loss": 1.0699033737182617e-05,
+      "reward": -0.9429489076137543,
+      "reward_std": 0.05583410756662488,
+      "rewards/cosine_scaled_reward": -0.47147445380687714,
+      "rewards/format_reward": 0.0,
+      "step": 48
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.028,
+      "kl": 0.00023242831230163574,
+      "learning_rate": 8.016368065618361e-06,
+      "loss": 9.488314390182495e-06,
+      "reward": -0.9678349643945694,
+      "reward_std": 0.031033652368932962,
+      "rewards/cosine_scaled_reward": -0.4839174821972847,
+      "rewards/format_reward": 0.0,
+      "step": 49
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.02857142857142857,
+      "kl": 0.0002980530261993408,
+      "learning_rate": 7.884636689049423e-06,
+      "loss": 1.1995434761047363e-05,
+      "reward": -0.92691770195961,
+      "reward_std": 0.06057529430836439,
+      "rewards/cosine_scaled_reward": -0.4634588584303856,
+      "rewards/format_reward": 0.0,
+      "step": 50
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.029142857142857144,
+      "kl": 0.0002821683883666992,
+      "learning_rate": 7.75e-06,
+      "loss": 1.093745231628418e-05,
+      "reward": -0.9709955900907516,
+      "reward_std": 0.023253681138157845,
+      "rewards/cosine_scaled_reward": -0.48549777269363403,
+      "rewards/format_reward": 0.0,
+      "step": 51
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.029714285714285714,
+      "kl": 0.0003021657466888428,
+      "learning_rate": 7.612622032536508e-06,
+      "loss": 1.2755393981933594e-05,
+      "reward": -0.932096004486084,
+      "reward_std": 0.0814181575551629,
+      "rewards/cosine_scaled_reward": -0.4660480171442032,
+      "rewards/format_reward": 0.0,
+      "step": 52
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.030285714285714287,
+      "kl": 0.0003120899200439453,
+      "learning_rate": 7.472670160550849e-06,
+      "loss": 1.1809170246124268e-05,
+      "reward": -0.8598470240831375,
+      "reward_std": 0.23317514825612307,
+      "rewards/cosine_scaled_reward": -0.42992351949214935,
+      "rewards/format_reward": 0.0,
+      "step": 53
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.030857142857142857,
+      "kl": 0.00031960010528564453,
+      "learning_rate": 7.330314893841102e-06,
+      "loss": 1.2435019016265869e-05,
+      "reward": -0.8917691111564636,
+      "reward_std": 0.16836319211870432,
+      "rewards/cosine_scaled_reward": -0.4458845555782318,
+      "rewards/format_reward": 0.0,
+      "step": 54
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.03142857142857143,
+      "kl": 0.0003180503845214844,
+      "learning_rate": 7.185729670371605e-06,
+      "loss": 1.2367963790893555e-05,
+      "reward": -0.9393985569477081,
+      "reward_std": 0.09125666646286845,
+      "rewards/cosine_scaled_reward": -0.46969927847385406,
+      "rewards/format_reward": 0.0,
+      "step": 55
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.032,
+      "kl": 0.00032332539558410645,
+      "learning_rate": 7.0390906449655104e-06,
+      "loss": 1.3027340173721313e-05,
+      "reward": -0.9556453377008438,
+      "reward_std": 0.03563447529450059,
+      "rewards/cosine_scaled_reward": -0.4778226688504219,
+      "rewards/format_reward": 0.0,
+      "step": 56
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.03257142857142857,
+      "kl": 0.00029200315475463867,
+      "learning_rate": 6.890576474687264e-06,
+      "loss": 1.173466444015503e-05,
+      "reward": -0.9533008933067322,
+      "reward_std": 0.047348865773528814,
+      "rewards/cosine_scaled_reward": -0.4766504615545273,
+      "rewards/format_reward": 0.0,
+      "step": 57
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.03314285714285714,
+      "kl": 0.00028020143508911133,
+      "learning_rate": 6.740368101176496e-06,
+      "loss": 1.0944902896881104e-05,
+      "reward": -0.8606822788715363,
+      "reward_std": 0.19956759549677372,
+      "rewards/cosine_scaled_reward": -0.4303411394357681,
+      "rewards/format_reward": 0.0,
+      "step": 58
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.03371428571428572,
+      "kl": 0.00034242868423461914,
+      "learning_rate": 6.588648530198505e-06,
+      "loss": 1.4327466487884521e-05,
+      "reward": -0.9435078203678131,
+      "reward_std": 0.028082083677873015,
+      "rewards/cosine_scaled_reward": -0.47175391018390656,
+      "rewards/format_reward": 0.0,
+      "step": 59
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.03428571428571429,
+      "kl": 0.0004596710205078125,
+      "learning_rate": 6.4356026086799176e-06,
+      "loss": 1.862645149230957e-05,
+      "reward": -0.8699476420879364,
+      "reward_std": 0.14623272977769375,
+      "rewards/cosine_scaled_reward": -0.4349738284945488,
+      "rewards/format_reward": 0.0,
+      "step": 60
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.03485714285714286,
+      "kl": 0.00041598081588745117,
+      "learning_rate": 6.281416799501188e-06,
+      "loss": 1.6495585441589355e-05,
+      "reward": -0.8786782920360565,
+      "reward_std": 0.14873503288254142,
+      "rewards/cosine_scaled_reward": -0.43933914601802826,
+      "rewards/format_reward": 0.0,
+      "step": 61
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.03542857142857143,
+      "kl": 0.0003428459167480469,
+      "learning_rate": 6.126278954320295e-06,
+      "loss": 1.4573335647583008e-05,
+      "reward": -0.9531020373106003,
+      "reward_std": 0.04140742728486657,
+      "rewards/cosine_scaled_reward": -0.47655102610588074,
+      "rewards/format_reward": 0.0,
+      "step": 62
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.036,
+      "kl": 0.00034630298614501953,
+      "learning_rate": 5.970378084704441e-06,
+      "loss": 1.3574957847595215e-05,
+      "reward": -0.9625549018383026,
+      "reward_std": 0.0323515310883522,
+      "rewards/cosine_scaled_reward": -0.4812774509191513,
+      "rewards/format_reward": 0.0,
+      "step": 63
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.036571428571428574,
+      "kl": 0.000436633825302124,
+      "learning_rate": 5.813904131848565e-06,
+      "loss": 1.7490237951278687e-05,
+      "reward": -0.9638274163007736,
+      "reward_std": 0.03202465921640396,
+      "rewards/cosine_scaled_reward": -0.4819137006998062,
+      "rewards/format_reward": 0.0,
+      "step": 64
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.037142857142857144,
+      "kl": 0.00037598609924316406,
+      "learning_rate": 5.657047735161256e-06,
+      "loss": 1.4990568161010742e-05,
+      "reward": -0.8853072375059128,
+      "reward_std": 0.11870932951569557,
+      "rewards/cosine_scaled_reward": -0.4426536113023758,
+      "rewards/format_reward": 0.0,
+      "step": 65
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.037714285714285714,
+      "kl": 0.000420987606048584,
+      "learning_rate": 5.500000000000001e-06,
+      "loss": 1.6763806343078613e-05,
+      "reward": -0.9631430506706238,
+      "reward_std": 0.027665999252349138,
+      "rewards/cosine_scaled_reward": -0.4815715327858925,
+      "rewards/format_reward": 0.0,
+      "step": 66
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.038285714285714284,
+      "kl": 0.0005689859390258789,
+      "learning_rate": 5.342952264838748e-06,
+      "loss": 2.3305416107177734e-05,
+      "reward": -0.9261669814586639,
+      "reward_std": 0.09819867718033493,
+      "rewards/cosine_scaled_reward": -0.46308349817991257,
+      "rewards/format_reward": 0.0,
+      "step": 67
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.038857142857142854,
+      "kl": 0.0006139874458312988,
+      "learning_rate": 5.186095868151436e-06,
+      "loss": 2.4430453777313232e-05,
+      "reward": -0.9322563856840134,
+      "reward_std": 0.06839151354506612,
+      "rewards/cosine_scaled_reward": -0.4661281928420067,
+      "rewards/format_reward": 0.0,
+      "step": 68
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.03942857142857143,
+      "kl": 0.0004551410675048828,
+      "learning_rate": 5.02962191529556e-06,
+      "loss": 1.753866672515869e-05,
+      "reward": -0.947336733341217,
+      "reward_std": 0.06602911371737719,
+      "rewards/cosine_scaled_reward": -0.4736683666706085,
+      "rewards/format_reward": 0.0,
+      "step": 69
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.04,
+      "kl": 0.0004698038101196289,
+      "learning_rate": 4.873721045679707e-06,
+      "loss": 1.9043684005737305e-05,
+      "reward": -0.881416529417038,
+      "reward_std": 0.15695088542997837,
+      "rewards/cosine_scaled_reward": -0.440708264708519,
+      "rewards/format_reward": 0.0,
+      "step": 70
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.04057142857142857,
+      "kl": 0.0005980134010314941,
+      "learning_rate": 4.718583200498814e-06,
+      "loss": 2.4437904357910156e-05,
+      "reward": -0.9552861750125885,
+      "reward_std": 0.04716450162231922,
+      "rewards/cosine_scaled_reward": -0.47764309495687485,
+      "rewards/format_reward": 0.0,
+      "step": 71
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.04114285714285714,
+      "kl": 0.00044411420822143555,
+      "learning_rate": 4.564397391320085e-06,
+      "loss": 1.781061291694641e-05,
+      "reward": -0.9175944626331329,
+      "reward_std": 0.07855925057083368,
+      "rewards/cosine_scaled_reward": -0.45879723131656647,
+      "rewards/format_reward": 0.0,
+      "step": 72
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.04171428571428572,
+      "kl": 0.0005142688751220703,
+      "learning_rate": 4.4113514698014955e-06,
+      "loss": 2.104230225086212e-05,
+      "reward": -0.9591091871261597,
+      "reward_std": 0.03032594360411167,
+      "rewards/cosine_scaled_reward": -0.47955460846424103,
+      "rewards/format_reward": 0.0,
+      "step": 73
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.04228571428571429,
+      "kl": 0.0004570484161376953,
+      "learning_rate": 4.259631898823504e-06,
+      "loss": 1.8224120140075684e-05,
+      "reward": -0.9418470114469528,
+      "reward_std": 0.09771440364420414,
+      "rewards/cosine_scaled_reward": -0.470923513174057,
+      "rewards/format_reward": 0.0,
+      "step": 74
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.04285714285714286,
+      "kl": 0.00042492151260375977,
+      "learning_rate": 4.109423525312738e-06,
+      "loss": 1.697242259979248e-05,
+      "reward": -0.9031769037246704,
+      "reward_std": 0.11431426065973938,
+      "rewards/cosine_scaled_reward": -0.4515884444117546,
+      "rewards/format_reward": 0.0,
+      "step": 75
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.04342857142857143,
+      "kl": 0.0003921389579772949,
+      "learning_rate": 3.960909355034491e-06,
+      "loss": 1.5683472156524658e-05,
+      "reward": -0.8710711896419525,
+      "reward_std": 0.16496195830404758,
+      "rewards/cosine_scaled_reward": -0.43553559482097626,
+      "rewards/format_reward": 0.0,
+      "step": 76
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.044,
+      "kl": 0.0005483031272888184,
+      "learning_rate": 3.8142703296283954e-06,
+      "loss": 2.2258609533309937e-05,
+      "reward": -0.8363375961780548,
+      "reward_std": 0.23763815127313137,
+      "rewards/cosine_scaled_reward": -0.418168805539608,
+      "rewards/format_reward": 0.0,
+      "step": 77
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.044571428571428574,
+      "kl": 0.0005096793174743652,
+      "learning_rate": 3.6696851061589e-06,
+      "loss": 2.06679105758667e-05,
+      "reward": -0.912532389163971,
+      "reward_std": 0.1374040930531919,
+      "rewards/cosine_scaled_reward": -0.45626621693372726,
+      "rewards/format_reward": 0.0,
+      "step": 78
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.045142857142857144,
+      "kl": 0.0007692575454711914,
+      "learning_rate": 3.527329839449152e-06,
+      "loss": 3.053247928619385e-05,
+      "reward": -0.9567261040210724,
+      "reward_std": 0.03342599933966994,
+      "rewards/cosine_scaled_reward": -0.478363037109375,
+      "rewards/format_reward": 0.0,
+      "step": 79
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.045714285714285714,
+      "kl": 0.0008780956268310547,
+      "learning_rate": 3.3873779674634932e-06,
+      "loss": 3.489106893539429e-05,
+      "reward": -0.8690381050109863,
+      "reward_std": 0.166263896971941,
+      "rewards/cosine_scaled_reward": -0.43451904505491257,
+      "rewards/format_reward": 0.0,
+      "step": 80
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.046285714285714284,
+      "kl": 0.000619053840637207,
+      "learning_rate": 3.2500000000000015e-06,
+      "loss": 2.2970139980316162e-05,
+      "reward": -0.9703131467103958,
+      "reward_std": 0.018532322952523828,
+      "rewards/cosine_scaled_reward": -0.4851565733551979,
+      "rewards/format_reward": 0.0,
+      "step": 81
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.046857142857142854,
+      "kl": 0.0005697011947631836,
+      "learning_rate": 3.115363310950579e-06,
+      "loss": 2.2605061531066895e-05,
+      "reward": -0.8376921266317368,
+      "reward_std": 0.21680933889001608,
+      "rewards/cosine_scaled_reward": -0.4188460633158684,
+      "rewards/format_reward": 0.0,
+      "step": 82
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.04742857142857143,
+      "kl": 0.0006507635116577148,
+      "learning_rate": 2.98363193438164e-06,
+      "loss": 2.580881118774414e-05,
+      "reward": -0.9194770008325577,
+      "reward_std": 0.07644858444109559,
+      "rewards/cosine_scaled_reward": -0.45973849296569824,
+      "rewards/format_reward": 0.0,
+      "step": 83
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.048,
+      "kl": 0.0005425214767456055,
+      "learning_rate": 2.854966364683872e-06,
+      "loss": 2.232193946838379e-05,
+      "reward": -0.921579897403717,
+      "reward_std": 0.07768060895614326,
+      "rewards/cosine_scaled_reward": -0.4607899561524391,
+      "rewards/format_reward": 0.0,
+      "step": 84
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.04857142857142857,
+      "kl": 0.0005598664283752441,
+      "learning_rate": 2.7295233610345384e-06,
+      "loss": 2.230703830718994e-05,
+      "reward": -0.8130292594432831,
+      "reward_std": 0.21775285061448812,
+      "rewards/cosine_scaled_reward": -0.40651462972164154,
+      "rewards/format_reward": 0.0,
+      "step": 85
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.04914285714285714,
+      "kl": 0.0005010366439819336,
+      "learning_rate": 2.607455756410573e-06,
+      "loss": 2.0213425159454346e-05,
+      "reward": -0.911712571978569,
+      "reward_std": 0.09181239921599627,
+      "rewards/cosine_scaled_reward": -0.4558562785387039,
+      "rewards/format_reward": 0.0,
+      "step": 86
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.04971428571428571,
+      "kl": 0.0005140304565429688,
+      "learning_rate": 2.4889122713851397e-06,
+      "loss": 2.09808349609375e-05,
+      "reward": -0.9401500076055527,
+      "reward_std": 0.04285714589059353,
+      "rewards/cosine_scaled_reward": -0.47007501125335693,
+      "rewards/format_reward": 0.0,
+      "step": 87
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.05028571428571429,
+      "kl": 0.0005880594253540039,
+      "learning_rate": 2.374037332934512e-06,
+      "loss": 2.3573637008666992e-05,
+      "reward": -0.8337626904249191,
+      "reward_std": 0.24908871483057737,
+      "rewards/cosine_scaled_reward": -0.41688134521245956,
+      "rewards/format_reward": 0.0,
+      "step": 88
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.05085714285714286,
+      "kl": 0.0005828142166137695,
+      "learning_rate": 2.262970898476071e-06,
+      "loss": 2.3312866687774658e-05,
+      "reward": -0.9419961720705032,
+      "reward_std": 0.06526243314146996,
+      "rewards/cosine_scaled_reward": -0.4709980934858322,
+      "rewards/format_reward": 0.0,
+      "step": 89
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.05142857142857143,
+      "kl": 0.0005592107772827148,
+      "learning_rate": 2.1558482853517257e-06,
+      "loss": 2.16066837310791e-05,
+      "reward": -0.9420932680368423,
+      "reward_std": 0.05442978721112013,
+      "rewards/cosine_scaled_reward": -0.47104664146900177,
+      "rewards/format_reward": 0.0,
+      "step": 90
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.052,
+      "kl": 0.0005729198455810547,
+      "learning_rate": 2.0528000059646e-06,
+      "loss": 2.3186206817626953e-05,
+      "reward": -0.8448977470397949,
+      "reward_std": 0.17764843348413706,
+      "rewards/cosine_scaled_reward": -0.42244888097047806,
+      "rewards/format_reward": 0.0,
+      "step": 91
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.052571428571428575,
+      "kl": 0.0006086826324462891,
+      "learning_rate": 1.953951608769752e-06,
+      "loss": 2.4143606424331665e-05,
+      "reward": -0.8687557131052017,
+      "reward_std": 0.13139949878677726,
+      "rewards/cosine_scaled_reward": -0.43437784910202026,
+      "rewards/format_reward": 0.0,
+      "step": 92
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.053142857142857144,
+      "kl": 0.0004508495330810547,
+      "learning_rate": 1.8594235253127373e-06,
+      "loss": 1.7457641661167145e-05,
+      "reward": -0.9452957063913345,
+      "reward_std": 0.044932478107512,
+      "rewards/cosine_scaled_reward": -0.4726478382945061,
+      "rewards/format_reward": 0.0,
+      "step": 93
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.053714285714285714,
+      "kl": 0.00058746337890625,
+      "learning_rate": 1.769330923502313e-06,
+      "loss": 2.3424625396728516e-05,
+      "reward": -0.9732565432786942,
+      "reward_std": 0.023667596746236086,
+      "rewards/cosine_scaled_reward": -0.48662828654050827,
+      "rewards/format_reward": 0.0,
+      "step": 94
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.054285714285714284,
+      "kl": 0.0006615519523620605,
+      "learning_rate": 1.6837835672960834e-06,
+      "loss": 2.664327621459961e-05,
+      "reward": -0.8294684588909149,
+      "reward_std": 0.21260959655046463,
+      "rewards/cosine_scaled_reward": -0.41473422944545746,
+      "rewards/format_reward": 0.0,
+      "step": 95
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.054857142857142854,
+      "kl": 0.0005936622619628906,
+      "learning_rate": 1.602885682970026e-06,
+      "loss": 2.310052514076233e-05,
+      "reward": -0.8817574381828308,
+      "reward_std": 0.14791762363165617,
+      "rewards/cosine_scaled_reward": -0.4408787190914154,
+      "rewards/format_reward": 0.0,
+      "step": 96
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.05542857142857143,
+      "kl": 0.0004170536994934082,
+      "learning_rate": 1.526735832134829e-06,
+      "loss": 1.5437602996826172e-05,
+      "reward": -0.9217518121004105,
+      "reward_std": 0.05548063712194562,
+      "rewards/cosine_scaled_reward": -0.46087589859962463,
+      "rewards/format_reward": 0.0,
+      "step": 97
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.056,
+      "kl": 0.0005135536193847656,
+      "learning_rate": 1.4554267916537495e-06,
+      "loss": 2.017989754676819e-05,
+      "reward": -0.9414444863796234,
+      "reward_std": 0.057337059173732996,
+      "rewards/cosine_scaled_reward": -0.4707222431898117,
+      "rewards/format_reward": 0.0,
+      "step": 98
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.05657142857142857,
+      "kl": 0.0004296302795410156,
+      "learning_rate": 1.389045440608296e-06,
+      "loss": 1.6748905181884766e-05,
+      "reward": -0.8684801608324051,
+      "reward_std": 0.18490357510745525,
+      "rewards/cosine_scaled_reward": -0.43424008786678314,
+      "rewards/format_reward": 0.0,
+      "step": 99
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 1024.0,
+      "epoch": 0.05714285714285714,
+      "kl": 0.0005539655685424805,
+      "learning_rate": 1.3276726544494572e-06,
+      "loss": 2.2009015083312988e-05,
+      "reward": -0.8967786431312561,
+      "reward_std": 0.12791539868339896,
+      "rewards/cosine_scaled_reward": -0.44838932156562805,
+      "rewards/format_reward": 0.0,
+      "step": 100
+    },
+    {
+      "epoch": 0.05714285714285714,
+      "step": 100,
       "total_flos": 0.0,
-      "train_loss": 4.023313522338867e-07,
-      "train_runtime": 537.7965,
-      "train_samples_per_second": 0.015,
-      "train_steps_per_second": 0.002
+      "train_loss": 0.0013520326372236013,
+      "train_runtime": 36067.9715,
+      "train_samples_per_second": 0.044,
+      "train_steps_per_second": 0.003
     }
   ],
   "logging_steps": 1,
-  "max_steps": 1,
+  "max_steps": 100,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 50,
@@ -50,7 +1337,7 @@
     }
   },
   "total_flos": 0.0,
-  "train_batch_size": 1,
+  "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null
 }

training_metrics.txt

@@ -1,6 +1,6 @@
-total_size_before (MB): 1455.72
-total_size_after (MB): 1445.40
-total_time (seconds): 544.56
-ram_peak (MB): 3195.88
-ram_consump (MB): 1477.19
-disk_storage (MB): 575.25
+total_size_before (MB): 3424.75
+total_size_after (MB): 3407.14
+total_time (seconds): 36073.19
+ram_peak (MB): 5853.69
+ram_consump (MB): 3485.15
+disk_storage (MB): 333.13