Model save

README.md

@@ -1,11 +1,9 @@
 ---
-base_model: Qwen/Qwen3.5-0.8B
-datasets: knoveleng/open-rs
+base_model: deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B
 library_name: transformers
 model_name: OpenRS-GRPO
 tags:
 - generated_from_trainer
-- open-r1
 - trl
 - grpo
 licence: license
@@ -13,7 +11,7 @@ licence: license
 
 # Model Card for OpenRS-GRPO
 
-This model is a fine-tuned version of [Qwen/Qwen3.5-0.8B](https://huggingface.co/Qwen/Qwen3.5-0.8B) on the [knoveleng/open-rs](https://huggingface.co/datasets/knoveleng/open-rs) dataset.
+This model is a fine-tuned version of [deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B).
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.010430806130170823,
-    "train_runtime": 9458.0577,
+    "train_loss": -3.18021047860384e-06,
+    "train_runtime": 16301.2017,
     "train_samples": 7000,
-    "train_samples_per_second": 0.034,
-    "train_steps_per_second": 0.002
+    "train_samples_per_second": 0.02,
+    "train_steps_per_second": 0.001
 }

step_metrics.csv

@@ -1,22 +1,22 @@
 step,epoch,loss,learning_rate,grad_norm,rewards/format_reward,rewards/cosine_scaled_reward,reward,reward_std,gpu_mem_alloc_mb,gpu_mem_peak_mb,step_time_sec
-1,0.0006,-1.2665987014770508e-07,0.0,,0.0,-0.3879377990961075,-0.7758755832910538,0.2861072635278106,1565.2,3646.7,476.39
-2,0.0011,2.2351741790771484e-08,5e-07,,0.0,-0.34302495419979095,-0.6860499083995819,0.3919920399785042,1565.2,3663.9,469.44
-3,0.0017,0.04199279844760895,1e-06,,0.0,-0.39361898601055145,-0.7872379571199417,0.28899660520255566,1565.2,3665.2,464.19
-4,0.0023,0.00015985965728759766,9.931634888554935e-07,,0.0,-0.39139123260974884,-0.7827824652194977,0.30724803544580936,1565.2,3665.5,473.28
-5,0.0029,0.016923315823078156,9.728616793536587e-07,,0.0,-0.3417773097753525,-0.6835546344518661,0.4394468888640404,1565.2,3666.6,475.57
-6,0.0034,0.00023673847317695618,9.397114317029974e-07,,0.0,-0.43544115871191025,-0.8708823472261429,0.1456776731647551,1565.2,3669.9,476.37
-7,0.004,5.0827860832214355e-05,8.9471999940354e-07,,0.0,-0.46910375356674194,-0.9382074922323227,0.08606540504842997,1565.2,3669.9,481.67
-8,0.0046,7.636845111846924e-05,8.392544243589427e-07,,0.0,-0.3953063264489174,-0.7906126528978348,0.18317685835063457,1565.2,3669.9,481.42
-9,0.0051,7.21365213394165e-05,7.75e-07,,0.0,-0.4495018497109413,-0.8990036994218826,0.1606585686095059,1565.2,3669.9,479.75
-10,0.0057,0.00016715750098228455,7.039090644965509e-07,,0.0,-0.4300354793667793,-0.860070988535881,0.17052607703953981,1565.2,3669.9,466.26
-11,0.0063,0.00013599544763565063,6.281416799501187e-07,,0.0,-0.4115590825676918,-0.8231181800365448,0.1259385095909238,1565.2,3669.9,472.08
-12,0.0069,5.13419508934021e-05,5.5e-07,,0.0,-0.4723722040653229,-0.9447444081306458,0.08061030774842948,1565.2,3669.9,471.88
-13,0.0074,0.07869705557823181,4.7185832004988133e-07,,0.0,-0.44672856479883194,-0.8934571295976639,0.15774485282599926,1565.2,3669.9,466.09
-14,0.008,0.027497582137584686,3.9609093550344907e-07,,0.0,-0.3920762911438942,-0.7841525673866272,0.2220854666084051,1565.2,3669.9,468.75
-15,0.0086,0.0004888176918029785,3.250000000000001e-07,,0.0,-0.3608057275414467,-0.7216114401817322,0.3453192347660661,1565.2,3669.9,462.49
-16,0.0091,0.00015526264905929565,2.6074557564105724e-07,,0.0,-0.4132692217826843,-0.8265384286642075,0.25778803089633584,1565.2,3669.9,476.11
-17,0.0097,0.02472507953643799,2.0528000059645995e-07,,0.0,-0.4350534752011299,-0.8701069504022598,0.18937412789091468,1565.2,3669.9,469.71
-18,0.0103,0.00023746490478515625,1.6028856829700258e-07,,0.0,-0.41003918647766113,-0.8200783580541611,0.26157089229673147,1565.2,3669.9,472.28
-19,0.0109,0.016675502061843872,1.2713832064634125e-07,,0.0,-0.41168487817049026,-0.8233697563409805,0.20013628248125315,1565.2,3669.9,477.29
-20,0.0114,0.00027292221784591675,1.068365111445064e-07,,0.0,-0.4022079259157181,-0.8044158518314362,0.2423506089253351,1565.2,3669.9,473.18
-20,0.0114,,,,,,,,1565.2,3669.9,477.0
+1,0.0006,2.980232238769531e-07,0.0,,0.0,-0.4581816643476486,-0.9163633286952972,0.07046629022806883,3645.2,6086.5,810.9
+2,0.0011,4.6566128730773926e-07,5e-07,,0.0,-0.47419849038124084,-0.9483969509601593,0.06443409714847803,3645.2,6119.0,813.84
+3,0.0017,-6.4373016357421875e-06,1e-06,,0.0,-0.47469519078731537,-0.9493903964757919,0.02546792710199952,3645.2,6120.5,814.88
+4,0.0023,-6.161630153656006e-06,9.931634888554935e-07,,0.0,-0.4800366908311844,-0.9600733816623688,0.03750546649098396,3645.2,6122.9,815.73
+5,0.0029,-1.1235475540161133e-05,9.728616793536587e-07,,0.0,-0.4610184580087662,-0.9220369160175323,0.05609214352443814,3645.2,6122.9,815.84
+6,0.0034,-5.6587159633636475e-06,9.397114317029974e-07,,0.0,-0.43115096539258957,-0.8623019307851791,0.19040754111483693,3645.2,6129.7,815.34
+7,0.004,-6.24731183052063e-06,8.9471999940354e-07,,0.0,-0.4677419885993004,-0.9354839473962784,0.056630742736160755,3645.2,6129.7,814.51
+8,0.0046,-5.133450031280518e-06,8.392544243589427e-07,,0.0,-0.4351673647761345,-0.8703347146511078,0.10057847108691931,3645.2,6129.7,815.12
+9,0.0051,-4.664063453674316e-06,7.75e-07,,0.0,-0.4807252585887909,-0.9614505171775818,0.0432720510289073,3645.2,6129.7,815.01
+10,0.0057,-2.2277235984802246e-06,7.039090644965509e-07,,0.0,-0.38343894481658936,-0.7668778896331787,0.3067741859704256,3645.2,6129.7,813.54
+11,0.0063,-1.996755599975586e-06,6.281416799501187e-07,,0.0,-0.2307990826666355,-0.461598165333271,0.3550597131252289,3645.2,6129.7,814.71
+12,0.0069,-1.2740492820739746e-06,5.5e-07,,0.0,-0.19471427984535694,-0.3894285596907139,0.27436650544404984,3645.2,6129.7,814.77
+13,0.0074,-2.5406479835510254e-06,4.7185832004988133e-07,,0.0,-0.2408045493066311,-0.4816090911626816,0.3208252266049385,3645.2,6129.7,814.34
+14,0.008,-1.5497207641601562e-06,3.9609093550344907e-07,,0.0,-0.35031646490097046,-0.7006329447031021,0.3634557966142893,3645.2,6129.7,814.91
+15,0.0086,-1.7210841178894043e-06,3.250000000000001e-07,,0.0,-0.2049925960600376,-0.40998518466949463,0.3375362530350685,3645.2,6129.7,815.43
+16,0.0091,-1.9073486328125e-06,2.6074557564105724e-07,,0.0,-0.17782340943813324,-0.3556468114256859,0.38361550495028496,3645.2,6129.7,816.11
+17,0.0097,-2.1010637283325195e-06,2.0528000059645995e-07,,0.0,-0.28106561303138733,-0.5621312409639359,0.371349073946476,3645.2,6129.7,815.13
+18,0.0103,-1.7527490854263306e-06,1.6028856829700258e-07,,0.0,-0.25671521946787834,-0.5134304240345955,0.34325383603572845,3645.2,6129.7,815.46
+19,0.0109,-2.980232238769531e-07,1.2713832064634125e-07,,0.0,-0.20568780414760113,-0.41137560456991196,0.28929552249610424,3645.2,6129.7,816.18
+20,0.0114,-1.460779458284378e-06,1.068365111445064e-07,,0.0,-0.31505120918154716,-0.6301024332642555,0.33287271670997143,3645.2,6129.7,815.67
+20,0.0114,,,,,,,,3645.2,6129.7,819.4

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.010430806130170823,
-    "train_runtime": 9458.0577,
+    "train_loss": -3.18021047860384e-06,
+    "train_runtime": 16301.2017,
     "train_samples": 7000,
-    "train_samples_per_second": 0.034,
-    "train_steps_per_second": 0.002
+    "train_samples_per_second": 0.02,
+    "train_steps_per_second": 0.001
 }

trainer_state.json

@@ -11,261 +11,261 @@
   "log_history": [
     {
       "clip_ratio": 0.0,
-      "completion_length": 453.625,
+      "completion_length": 1024.0,
       "epoch": 0.0005714285714285715,
       "kl": 0.0,
       "learning_rate": 0.0,
-      "loss": -1.2665987014770508e-07,
-      "reward": -0.7758755832910538,
-      "reward_std": 0.2861072635278106,
-      "rewards/cosine_scaled_reward": -0.3879377990961075,
+      "loss": 2.980232238769531e-07,
+      "reward": -0.9163633286952972,
+      "reward_std": 0.07046629022806883,
+      "rewards/cosine_scaled_reward": -0.4581816643476486,
       "rewards/format_reward": 0.0,
       "step": 1
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 512.0,
+      "completion_length": 1024.0,
       "epoch": 0.001142857142857143,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": 2.2351741790771484e-08,
-      "reward": -0.6860499083995819,
-      "reward_std": 0.3919920399785042,
-      "rewards/cosine_scaled_reward": -0.34302495419979095,
+      "loss": 4.6566128730773926e-07,
+      "reward": -0.9483969509601593,
+      "reward_std": 0.06443409714847803,
+      "rewards/cosine_scaled_reward": -0.47419849038124084,
       "rewards/format_reward": 0.0,
       "step": 2
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 494.9375,
+      "completion_length": 1024.0,
       "epoch": 0.0017142857142857142,
-      "kl": 0.00760650634765625,
+      "kl": -0.00019288063049316406,
       "learning_rate": 1e-06,
-      "loss": 0.04199279844760895,
-      "reward": -0.7872379571199417,
-      "reward_std": 0.28899660520255566,
-      "rewards/cosine_scaled_reward": -0.39361898601055145,
+      "loss": -6.4373016357421875e-06,
+      "reward": -0.9493903964757919,
+      "reward_std": 0.02546792710199952,
+      "rewards/cosine_scaled_reward": -0.47469519078731537,
       "rewards/format_reward": 0.0,
       "step": 3
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 484.875,
+      "completion_length": 1024.0,
       "epoch": 0.002285714285714286,
-      "kl": 0.00399017333984375,
+      "kl": -0.00016021728515625,
       "learning_rate": 9.931634888554935e-07,
-      "loss": 0.00015985965728759766,
-      "reward": -0.7827824652194977,
-      "reward_std": 0.30724803544580936,
-      "rewards/cosine_scaled_reward": -0.39139123260974884,
+      "loss": -6.161630153656006e-06,
+      "reward": -0.9600733816623688,
+      "reward_std": 0.03750546649098396,
+      "rewards/cosine_scaled_reward": -0.4800366908311844,
       "rewards/format_reward": 0.0,
       "step": 4
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 498.125,
+      "completion_length": 1024.0,
       "epoch": 0.002857142857142857,
-      "kl": 0.010009765625,
+      "kl": -0.00026488304138183594,
       "learning_rate": 9.728616793536587e-07,
-      "loss": 0.016923315823078156,
-      "reward": -0.6835546344518661,
-      "reward_std": 0.4394468888640404,
-      "rewards/cosine_scaled_reward": -0.3417773097753525,
+      "loss": -1.1235475540161133e-05,
+      "reward": -0.9220369160175323,
+      "reward_std": 0.05609214352443814,
+      "rewards/cosine_scaled_reward": -0.4610184580087662,
       "rewards/format_reward": 0.0,
       "step": 5
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 512.0,
+      "completion_length": 1024.0,
       "epoch": 0.0034285714285714284,
-      "kl": 0.0059185028076171875,
+      "kl": -0.000133514404296875,
       "learning_rate": 9.397114317029974e-07,
-      "loss": 0.00023673847317695618,
-      "reward": -0.8708823472261429,
-      "reward_std": 0.1456776731647551,
-      "rewards/cosine_scaled_reward": -0.43544115871191025,
+      "loss": -5.6587159633636475e-06,
+      "reward": -0.8623019307851791,
+      "reward_std": 0.19040754111483693,
+      "rewards/cosine_scaled_reward": -0.43115096539258957,
       "rewards/format_reward": 0.0,
       "step": 6
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 512.0,
+      "completion_length": 1024.0,
       "epoch": 0.004,
-      "kl": 0.001270294189453125,
+      "kl": -0.00017333030700683594,
       "learning_rate": 8.9471999940354e-07,
-      "loss": 5.0827860832214355e-05,
-      "reward": -0.9382074922323227,
-      "reward_std": 0.08606540504842997,
-      "rewards/cosine_scaled_reward": -0.46910375356674194,
+      "loss": -6.24731183052063e-06,
+      "reward": -0.9354839473962784,
+      "reward_std": 0.056630742736160755,
+      "rewards/cosine_scaled_reward": -0.4677419885993004,
       "rewards/format_reward": 0.0,
       "step": 7
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 480.625,
+      "completion_length": 1024.0,
       "epoch": 0.004571428571428572,
-      "kl": 0.0019054412841796875,
+      "kl": -0.00012612342834472656,
       "learning_rate": 8.392544243589427e-07,
-      "loss": 7.636845111846924e-05,
-      "reward": -0.7906126528978348,
-      "reward_std": 0.18317685835063457,
-      "rewards/cosine_scaled_reward": -0.3953063264489174,
+      "loss": -5.133450031280518e-06,
+      "reward": -0.8703347146511078,
+      "reward_std": 0.10057847108691931,
+      "rewards/cosine_scaled_reward": -0.4351673647761345,
       "rewards/format_reward": 0.0,
       "step": 8
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 512.0,
+      "completion_length": 1024.0,
       "epoch": 0.005142857142857143,
-      "kl": 0.0017995834350585938,
+      "kl": -9.72747802734375e-05,
       "learning_rate": 7.75e-07,
-      "loss": 7.21365213394165e-05,
-      "reward": -0.8990036994218826,
-      "reward_std": 0.1606585686095059,
-      "rewards/cosine_scaled_reward": -0.4495018497109413,
+      "loss": -4.664063453674316e-06,
+      "reward": -0.9614505171775818,
+      "reward_std": 0.0432720510289073,
+      "rewards/cosine_scaled_reward": -0.4807252585887909,
       "rewards/format_reward": 0.0,
       "step": 9
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 512.0,
+      "completion_length": 1024.0,
       "epoch": 0.005714285714285714,
-      "kl": 0.0041675567626953125,
+      "kl": -5.555152893066406e-05,
       "learning_rate": 7.039090644965509e-07,
-      "loss": 0.00016715750098228455,
-      "reward": -0.860070988535881,
-      "reward_std": 0.17052607703953981,
-      "rewards/cosine_scaled_reward": -0.4300354793667793,
+      "loss": -2.2277235984802246e-06,
+      "reward": -0.7668778896331787,
+      "reward_std": 0.3067741859704256,
+      "rewards/cosine_scaled_reward": -0.38343894481658936,
       "rewards/format_reward": 0.0,
       "step": 10
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 477.125,
+      "completion_length": 1024.0,
       "epoch": 0.006285714285714286,
-      "kl": 0.003407001495361328,
+      "kl": -4.9114227294921875e-05,
       "learning_rate": 6.281416799501187e-07,
-      "loss": 0.00013599544763565063,
-      "reward": -0.8231181800365448,
-      "reward_std": 0.1259385095909238,
-      "rewards/cosine_scaled_reward": -0.4115590825676918,
+      "loss": -1.996755599975586e-06,
+      "reward": -0.461598165333271,
+      "reward_std": 0.3550597131252289,
+      "rewards/cosine_scaled_reward": -0.2307990826666355,
       "rewards/format_reward": 0.0,
       "step": 11
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 512.0,
+      "completion_length": 1024.0,
       "epoch": 0.006857142857142857,
-      "kl": 0.0012102127075195312,
+      "kl": -3.147125244140625e-05,
       "learning_rate": 5.5e-07,
-      "loss": 5.13419508934021e-05,
-      "reward": -0.9447444081306458,
-      "reward_std": 0.08061030774842948,
-      "rewards/cosine_scaled_reward": -0.4723722040653229,
+      "loss": -1.2740492820739746e-06,
+      "reward": -0.3894285596907139,
+      "reward_std": 0.27436650544404984,
+      "rewards/cosine_scaled_reward": -0.19471427984535694,
       "rewards/format_reward": 0.0,
       "step": 12
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 487.9375,
+      "completion_length": 1024.0,
       "epoch": 0.0074285714285714285,
-      "kl": 0.00119781494140625,
+      "kl": -6.198883056640625e-05,
       "learning_rate": 4.7185832004988133e-07,
-      "loss": 0.07869705557823181,
-      "reward": -0.8934571295976639,
-      "reward_std": 0.15774485282599926,
-      "rewards/cosine_scaled_reward": -0.44672856479883194,
+      "loss": -2.5406479835510254e-06,
+      "reward": -0.4816090911626816,
+      "reward_std": 0.3208252266049385,
+      "rewards/cosine_scaled_reward": -0.2408045493066311,
       "rewards/format_reward": 0.0,
       "step": 13
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 466.625,
+      "completion_length": 1024.0,
       "epoch": 0.008,
-      "kl": 0.004004955291748047,
+      "kl": -4.029273986816406e-05,
       "learning_rate": 3.9609093550344907e-07,
-      "loss": 0.027497582137584686,
-      "reward": -0.7841525673866272,
-      "reward_std": 0.2220854666084051,
-      "rewards/cosine_scaled_reward": -0.3920762911438942,
+      "loss": -1.5497207641601562e-06,
+      "reward": -0.7006329447031021,
+      "reward_std": 0.3634557966142893,
+      "rewards/cosine_scaled_reward": -0.35031646490097046,
       "rewards/format_reward": 0.0,
       "step": 14
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 497.5,
+      "completion_length": 1024.0,
       "epoch": 0.008571428571428572,
-      "kl": 0.012208938598632812,
+      "kl": -4.410743713378906e-05,
       "learning_rate": 3.250000000000001e-07,
-      "loss": 0.0004888176918029785,
-      "reward": -0.7216114401817322,
-      "reward_std": 0.3453192347660661,
-      "rewards/cosine_scaled_reward": -0.3608057275414467,
+      "loss": -1.7210841178894043e-06,
+      "reward": -0.40998518466949463,
+      "reward_std": 0.3375362530350685,
+      "rewards/cosine_scaled_reward": -0.2049925960600376,
       "rewards/format_reward": 0.0,
       "step": 15
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 499.0,
+      "completion_length": 1024.0,
       "epoch": 0.009142857142857144,
-      "kl": 0.003875732421875,
+      "kl": -4.8160552978515625e-05,
       "learning_rate": 2.6074557564105724e-07,
-      "loss": 0.00015526264905929565,
-      "reward": -0.8265384286642075,
-      "reward_std": 0.25778803089633584,
-      "rewards/cosine_scaled_reward": -0.4132692217826843,
+      "loss": -1.9073486328125e-06,
+      "reward": -0.3556468114256859,
+      "reward_std": 0.38361550495028496,
+      "rewards/cosine_scaled_reward": -0.17782340943813324,
       "rewards/format_reward": 0.0,
       "step": 16
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 501.5,
+      "completion_length": 1024.0,
       "epoch": 0.009714285714285713,
-      "kl": 0.004252910614013672,
+      "kl": -5.2928924560546875e-05,
       "learning_rate": 2.0528000059645995e-07,
-      "loss": 0.02472507953643799,
-      "reward": -0.8701069504022598,
-      "reward_std": 0.18937412789091468,
-      "rewards/cosine_scaled_reward": -0.4350534752011299,
+      "loss": -2.1010637283325195e-06,
+      "reward": -0.5621312409639359,
+      "reward_std": 0.371349073946476,
+      "rewards/cosine_scaled_reward": -0.28106561303138733,
       "rewards/format_reward": 0.0,
       "step": 17
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 506.625,
+      "completion_length": 1024.0,
       "epoch": 0.010285714285714285,
-      "kl": 0.005939483642578125,
+      "kl": -4.3392181396484375e-05,
       "learning_rate": 1.6028856829700258e-07,
-      "loss": 0.00023746490478515625,
-      "reward": -0.8200783580541611,
-      "reward_std": 0.26157089229673147,
-      "rewards/cosine_scaled_reward": -0.41003918647766113,
+      "loss": -1.7527490854263306e-06,
+      "reward": -0.5134304240345955,
+      "reward_std": 0.34325383603572845,
+      "rewards/cosine_scaled_reward": -0.25671521946787834,
       "rewards/format_reward": 0.0,
       "step": 18
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 446.25,
+      "completion_length": 1024.0,
       "epoch": 0.010857142857142857,
-      "kl": 0.002822399139404297,
+      "kl": -7.867813110351562e-06,
       "learning_rate": 1.2713832064634125e-07,
-      "loss": 0.016675502061843872,
-      "reward": -0.8233697563409805,
-      "reward_std": 0.20013628248125315,
-      "rewards/cosine_scaled_reward": -0.41168487817049026,
+      "loss": -2.980232238769531e-07,
+      "reward": -0.41137560456991196,
+      "reward_std": 0.28929552249610424,
+      "rewards/cosine_scaled_reward": -0.20568780414760113,
       "rewards/format_reward": 0.0,
       "step": 19
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 496.125,
+      "completion_length": 1024.0,
       "epoch": 0.011428571428571429,
-      "kl": 0.00678253173828125,
+      "kl": -3.4809112548828125e-05,
       "learning_rate": 1.068365111445064e-07,
-      "loss": 0.00027292221784591675,
-      "reward": -0.8044158518314362,
-      "reward_std": 0.2423506089253351,
-      "rewards/cosine_scaled_reward": -0.4022079259157181,
+      "loss": -1.460779458284378e-06,
+      "reward": -0.6301024332642555,
+      "reward_std": 0.33287271670997143,
+      "rewards/cosine_scaled_reward": -0.31505120918154716,
       "rewards/format_reward": 0.0,
       "step": 20
     },
@@ -273,10 +273,10 @@
       "epoch": 0.011428571428571429,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 0.010430806130170823,
-      "train_runtime": 9458.0577,
-      "train_samples_per_second": 0.034,
-      "train_steps_per_second": 0.002
+      "train_loss": -3.18021047860384e-06,
+      "train_runtime": 16301.2017,
+      "train_samples_per_second": 0.02,
+      "train_steps_per_second": 0.001
     }
   ],
   "logging_steps": 1,

training_metrics.txt

@@ -1,6 +1,6 @@
-total_size_before (MB): 1455.72
-total_size_after (MB): 1445.40
-total_time (seconds): 9466.33
-ram_peak (MB): 3499.90
-ram_consump (MB): 1492.71
-disk_storage (MB): 616.56
+total_size_before (MB): 3424.75
+total_size_after (MB): 3407.14
+total_time (seconds): 16314.67
+ram_peak (MB): 5845.76
+ram_consump (MB): 3476.35
+disk_storage (MB): 180.81