Upload folder using huggingface_hub

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3da59759c1875fd9d7d0ebb082b52db9000f9ac5d6b37f76aead1f2af1848616
+oid sha256:90b53e3bd441110af3468dcc115ce68ffb693b44039e909d83d07496efb0d65a
 size 1976163472

optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5c4df16f1ca5833e0ad078a50b20910a45236bf8d3effd593dedb758ff080f85
+oid sha256:3d8a8dea3f9a14bc24f1378c17cd6eafe82c9fe590b6d03205ad8bf8befe2181
 size 3952509771

rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:781001445c9573ba228e84eb85acaacdf2f1c808a17c7a2b5471ec7b2eb6e333
+oid sha256:226146a4f4582e8746c8d09d92bc39c0215488d8ce29ce2dc28d5cb9ba8bb35c
 size 14581

scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4c8a76b64655ee1b513c2a00a0986b971a57367042b28265cea7ecb8a4d5ebe1
+oid sha256:09ecd6d2a95eec387a7a8a4e2c2c7824ffd785e4eeeb2e4c69a237465107c4f2
 size 1465

trainer_state.json

@@ -2,9 +2,9 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.078125,
+  "epoch": 0.25,
   "eval_steps": 500,
-  "global_step": 20,
+  "global_step": 100,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -15,31 +15,223 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.875,
-      "completions/max_length": 256.0,
-      "completions/max_terminated_length": 98.1,
-      "completions/mean_length": 254.225,
-      "completions/mean_terminated_length": 96.75,
-      "completions/min_length": 249.0,
-      "completions/min_terminated_length": 95.4,
-      "entropy": 0.4954624392092228,
-      "epoch": 0.0390625,
-      "frac_reward_zero_std": 0.1,
-      "grad_norm": 4.894988536834717,
-      "learning_rate": 5.5e-07,
-      "loss": -0.0063,
-      "num_tokens": 11137.0,
-      "reward": 0.5624999955296517,
-      "reward_std": 0.3298858031630516,
-      "rewards/reward_helper/mean": 0.5624999955296517,
-      "rewards/reward_helper/std": 0.32988580912351606,
-      "sampling/importance_sampling_ratio/max": 1.3241658806800842,
-      "sampling/importance_sampling_ratio/mean": 0.9233232378959656,
-      "sampling/importance_sampling_ratio/min": 0.5554532319307327,
-      "sampling/sampling_logp_difference/max": 0.1697020173072815,
-      "sampling/sampling_logp_difference/mean": 0.008749492838978768,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1.6,
+      "completions/max_terminated_length": 1.6,
+      "completions/mean_length": 1.15,
+      "completions/mean_terminated_length": 1.15,
+      "completions/min_length": 1.0,
+      "completions/min_terminated_length": 1.0,
+      "entropy": 0.44410082772374154,
+      "epoch": 0.025,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0,
+      "learning_rate": 9.1e-07,
+      "loss": 0.0,
+      "num_tokens": 3586.0,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/reward_helper/mean": 0.0,
+      "rewards/reward_helper/std": 0.0,
+      "sampling/importance_sampling_ratio/max": 1.0006482005119324,
+      "sampling/importance_sampling_ratio/mean": 0.9993368148803711,
+      "sampling/importance_sampling_ratio/min": 0.9954026460647583,
+      "sampling/sampling_logp_difference/max": 0.00986330844461918,
+      "sampling/sampling_logp_difference/mean": 0.004519483260810375,
       "step": 10,
-      "step_time": 8.9995359561
+      "step_time": 5.278360854899991
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.075,
+      "completions/max_length": 52.0,
+      "completions/max_terminated_length": 1.0,
+      "completions/mean_length": 20.125,
+      "completions/mean_terminated_length": 1.0,
+      "completions/min_length": 1.0,
+      "completions/min_terminated_length": 1.0,
+      "entropy": 0.5618356347084046,
+      "epoch": 0.05,
+      "frac_reward_zero_std": 0.8,
+      "grad_norm": 0.0,
+      "learning_rate": 8.1e-07,
+      "loss": -0.0875,
+      "num_tokens": 8279.0,
+      "reward": 0.015000000596046448,
+      "reward_std": 0.030000001192092896,
+      "rewards/reward_helper/mean": 0.015000000596046448,
+      "rewards/reward_helper/std": 0.030000004172325134,
+      "sampling/importance_sampling_ratio/max": 1.0033257365226746,
+      "sampling/importance_sampling_ratio/mean": 0.9744328677654266,
+      "sampling/importance_sampling_ratio/min": 0.8951155811548233,
+      "sampling/sampling_logp_difference/max": 0.039345417730510236,
+      "sampling/sampling_logp_difference/mean": 0.004816015157848597,
+      "step": 20,
+      "step_time": 1.230997591900001
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.025,
+      "completions/max_length": 46.5,
+      "completions/max_terminated_length": 21.0,
+      "completions/mean_length": 12.525,
+      "completions/mean_terminated_length": 6.15,
+      "completions/min_length": 1.0,
+      "completions/min_terminated_length": 1.0,
+      "entropy": 0.5129512194544077,
+      "epoch": 0.075,
+      "frac_reward_zero_std": 0.9,
+      "grad_norm": 0.0,
+      "learning_rate": 7.1e-07,
+      "loss": -0.0936,
+      "num_tokens": 12500.0,
+      "reward": 0.007500000298023224,
+      "reward_std": 0.015000000596046448,
+      "rewards/reward_helper/mean": 0.007500000298023224,
+      "rewards/reward_helper/std": 0.015000002086162567,
+      "sampling/importance_sampling_ratio/max": 1.0458860635757445,
+      "sampling/importance_sampling_ratio/mean": 1.0027949154376983,
+      "sampling/importance_sampling_ratio/min": 0.9623779714107513,
+      "sampling/sampling_logp_difference/max": 0.040722915157675746,
+      "sampling/sampling_logp_difference/mean": 0.006484306254424155,
+      "step": 30,
+      "step_time": 1.29722429359999
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.125,
+      "completions/max_length": 80.8,
+      "completions/max_terminated_length": 4.3,
+      "completions/mean_length": 33.725,
+      "completions/mean_terminated_length": 1.85,
+      "completions/min_length": 1.0,
+      "completions/min_terminated_length": 1.0,
+      "entropy": 0.8639558412134647,
+      "epoch": 0.1,
+      "frac_reward_zero_std": 0.7,
+      "grad_norm": 0.0,
+      "learning_rate": 6.1e-07,
+      "loss": -0.3408,
+      "num_tokens": 17421.0,
+      "reward": 0.017500000633299352,
+      "reward_std": 0.035000001266598704,
+      "rewards/reward_helper/mean": 0.017500000633299352,
+      "rewards/reward_helper/std": 0.03500000424683094,
+      "sampling/importance_sampling_ratio/max": 1.1232336461544037,
+      "sampling/importance_sampling_ratio/mean": 1.0079659342765808,
+      "sampling/importance_sampling_ratio/min": 0.9237572312355041,
+      "sampling/sampling_logp_difference/max": 0.08927474431693554,
+      "sampling/sampling_logp_difference/mean": 0.009856639616191386,
+      "step": 40,
+      "step_time": 1.325083582900001
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.175,
+      "completions/max_length": 143.6,
+      "completions/max_terminated_length": 55.9,
+      "completions/mean_length": 59.35,
+      "completions/mean_terminated_length": 15.916666793823243,
+      "completions/min_length": 1.0,
+      "completions/min_terminated_length": 1.0,
+      "entropy": 1.408535435795784,
+      "epoch": 0.125,
+      "frac_reward_zero_std": 0.6,
+      "grad_norm": 0.0,
+      "learning_rate": 5.1e-07,
+      "loss": -0.5045,
+      "num_tokens": 22655.0,
+      "reward": 0.025000000931322576,
+      "reward_std": 0.05000000186264515,
+      "rewards/reward_helper/mean": 0.025000000931322576,
+      "rewards/reward_helper/std": 0.05000000633299351,
+      "sampling/importance_sampling_ratio/max": 1.2991383910179137,
+      "sampling/importance_sampling_ratio/mean": 1.045933473110199,
+      "sampling/importance_sampling_ratio/min": 0.9206243932247162,
+      "sampling/sampling_logp_difference/max": 0.10195044428110123,
+      "sampling/sampling_logp_difference/mean": 0.011015815567225218,
+      "step": 50,
+      "step_time": 1.4795691685999999
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.15,
+      "completions/max_length": 119.2,
+      "completions/max_terminated_length": 68.1,
+      "completions/mean_length": 57.775,
+      "completions/mean_terminated_length": 19.425,
+      "completions/min_length": 26.5,
+      "completions/min_terminated_length": 0.9,
+      "entropy": 1.288734181970358,
+      "epoch": 0.15,
+      "frac_reward_zero_std": 0.6,
+      "grad_norm": 6.278634548187256,
+      "learning_rate": 4.0999999999999994e-07,
+      "loss": -0.2518,
+      "num_tokens": 28242.0,
+      "reward": 0.0325000012293458,
+      "reward_std": 0.05232050903141498,
+      "rewards/reward_helper/mean": 0.0325000012293458,
+      "rewards/reward_helper/std": 0.052320512011647224,
+      "sampling/importance_sampling_ratio/max": 1.1803278803825379,
+      "sampling/importance_sampling_ratio/mean": 1.020902580022812,
+      "sampling/importance_sampling_ratio/min": 0.8570977985858917,
+      "sampling/sampling_logp_difference/max": 0.08151691369712352,
+      "sampling/sampling_logp_difference/mean": 0.010703828372061252,
+      "step": 60,
+      "step_time": 1.4942365265000093
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.2,
+      "completions/max_length": 187.8,
+      "completions/max_terminated_length": 80.4,
+      "completions/mean_length": 76.725,
+      "completions/mean_terminated_length": 33.31666717529297,
+      "completions/min_length": 1.0,
+      "completions/min_terminated_length": 1.0,
+      "entropy": 2.0541629776358605,
+      "epoch": 0.175,
+      "frac_reward_zero_std": 0.4,
+      "grad_norm": 10.47471809387207,
+      "learning_rate": 3.1e-07,
+      "loss": -0.4418,
+      "num_tokens": 35115.0,
+      "reward": 0.05500000212341547,
+      "reward_std": 0.08464101739227772,
+      "rewards/reward_helper/mean": 0.05500000212341547,
+      "rewards/reward_helper/std": 0.08464102186262608,
+      "sampling/importance_sampling_ratio/max": 1.1099666595458983,
+      "sampling/importance_sampling_ratio/mean": 0.9488288760185242,
+      "sampling/importance_sampling_ratio/min": 0.7294621288776397,
+      "sampling/sampling_logp_difference/max": 0.14803548753261567,
+      "sampling/sampling_logp_difference/mean": 0.014868053188547493,
+      "step": 70,
+      "step_time": 1.6123995632000174
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -47,38 +239,102 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.9,
+      "completions/clipped_ratio": 0.5,
+      "completions/max_length": 230.5,
+      "completions/max_terminated_length": 85.9,
+      "completions/mean_length": 149.775,
+      "completions/mean_terminated_length": 52.43333396911621,
+      "completions/min_length": 83.3,
+      "completions/min_terminated_length": 32.1,
+      "entropy": 2.11170057579875,
+      "epoch": 0.2,
+      "frac_reward_zero_std": 0.4,
+      "grad_norm": 5.259692668914795,
+      "learning_rate": 2.0999999999999997e-07,
+      "loss": -0.2111,
+      "num_tokens": 43822.0,
+      "reward": 0.08250000271946192,
+      "reward_std": 0.10280726440250873,
+      "rewards/reward_helper/mean": 0.08250000271946192,
+      "rewards/reward_helper/std": 0.10280726738274097,
+      "sampling/importance_sampling_ratio/max": 1.539601093530655,
+      "sampling/importance_sampling_ratio/mean": 1.1134967446327209,
+      "sampling/importance_sampling_ratio/min": 0.7393792182207107,
+      "sampling/sampling_logp_difference/max": 0.1634996145963669,
+      "sampling/sampling_logp_difference/mean": 0.014190209750086068,
+      "step": 80,
+      "step_time": 1.6749743500999899
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.65,
       "completions/max_length": 256.0,
-      "completions/max_terminated_length": 67.8,
-      "completions/mean_length": 252.8,
-      "completions/mean_terminated_length": 66.4,
-      "completions/min_length": 244.2,
-      "completions/min_terminated_length": 65.0,
-      "entropy": 0.5851528078317643,
-      "epoch": 0.078125,
+      "completions/max_terminated_length": 76.9,
+      "completions/mean_length": 196.8,
+      "completions/mean_terminated_length": 67.38333435058594,
+      "completions/min_length": 83.5,
+      "completions/min_terminated_length": 57.9,
+      "entropy": 1.6522414579987525,
+      "epoch": 0.225,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 6.119882583618164,
+      "learning_rate": 1.0999999999999999e-07,
+      "loss": -0.2799,
+      "num_tokens": 55182.0,
+      "reward": 0.19750000536441803,
+      "reward_std": 0.22683281898498536,
+      "rewards/reward_helper/mean": 0.19750000536441803,
+      "rewards/reward_helper/std": 0.22683282494544982,
+      "sampling/importance_sampling_ratio/max": 1.3945529162883759,
+      "sampling/importance_sampling_ratio/mean": 0.9089209794998169,
+      "sampling/importance_sampling_ratio/min": 0.5628022223711013,
+      "sampling/sampling_logp_difference/max": 0.1890378475189209,
+      "sampling/sampling_logp_difference/mean": 0.0153791518881917,
+      "step": 90,
+      "step_time": 1.7072683768000103
+    },
+    {
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.6,
+      "completions/max_length": 255.0,
+      "completions/max_terminated_length": 120.7,
+      "completions/mean_length": 197.3,
+      "completions/mean_terminated_length": 87.23333358764648,
+      "completions/min_length": 112.7,
+      "completions/min_terminated_length": 61.5,
+      "entropy": 1.7242027707397938,
+      "epoch": 0.25,
       "frac_reward_zero_std": 0.1,
-      "grad_norm": 5.2831268310546875,
-      "learning_rate": 5e-08,
-      "loss": 0.0046,
-      "num_tokens": 22089.0,
-      "reward": 0.6474999994039535,
-      "reward_std": 0.28331912606954573,
-      "rewards/reward_helper/mean": 0.6474999994039535,
-      "rewards/reward_helper/std": 0.28331913203001025,
-      "sampling/importance_sampling_ratio/max": 1.5296368539333343,
-      "sampling/importance_sampling_ratio/mean": 1.074506050348282,
-      "sampling/importance_sampling_ratio/min": 0.6820479333400726,
-      "sampling/sampling_logp_difference/max": 0.1986921548843384,
-      "sampling/sampling_logp_difference/mean": 0.009845592267811298,
-      "step": 20,
-      "step_time": 5.0707887986999935
+      "grad_norm": 10.829450607299805,
+      "learning_rate": 1e-08,
+      "loss": -0.1902,
+      "num_tokens": 65950.0,
+      "reward": 0.11250000465661288,
+      "reward_std": 0.13696152605116368,
+      "rewards/reward_helper/mean": 0.11250000465661288,
+      "rewards/reward_helper/std": 0.13696153201162814,
+      "sampling/importance_sampling_ratio/max": 1.7122498869895935,
+      "sampling/importance_sampling_ratio/mean": 1.096574491262436,
+      "sampling/importance_sampling_ratio/min": 0.6619945377111435,
+      "sampling/sampling_logp_difference/max": 0.15592530369758606,
+      "sampling/sampling_logp_difference/mean": 0.015373438503593207,
+      "step": 100,
+      "step_time": 1.7581579990000022
     }
   ],
   "logging_steps": 10,
-  "max_steps": 20,
-  "num_input_tokens_seen": 22089,
+  "max_steps": 100,
+  "num_input_tokens_seen": 65950,
   "num_train_epochs": 1,
-  "save_steps": 5,
+  "save_steps": 10,
   "stateful_callbacks": {
     "TrainerControl": {
       "args": {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7c4dd33f5ae906b6d61fb5afde8165f9faf15dcdf2d9258ede16d3311b4cb607
-size 7633
+oid sha256:5ad22a28d8d4f0f2c1ffc0f71a4ce78d60e6521cf26c498c7b97024080ea428d
+size 7697