full-state checkpoint 40-percent (step 37)

ckpt-40-percent/adapter_config.json

@@ -22,7 +22,7 @@
   "loftq_config": {},
   "lora_alpha": 128,
   "lora_bias": false,
-  "lora_dropout": 0.0,
+  "lora_dropout": 0,
   "megatron_config": null,
   "megatron_core": "megatron.core",
   "modules_to_save": null,
@@ -33,13 +33,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "q_proj",
+    "o_proj",
+    "v_proj",
     "k_proj",
+    "down_proj",
     "up_proj",
-    "o_proj",
     "gate_proj",
-    "v_proj",
-    "down_proj"
+    "q_proj"
   ],
   "target_parameters": null,
   "task_type": "CAUSAL_LM",

ckpt-40-percent/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4193948521a74caee87f9af27bb89976d5850e03f214624ecea3d1591794fe3c
+oid sha256:547c520e9f6018dcc04e0887a68f66d4b548b65f8ba5326a187b3ce97736ffd8
 size 528550256

ckpt-40-percent/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d7474bafbe60682e7d3d2158a792c18fcb041721f237de832859e8b832bf9f2b
+oid sha256:7cead732039a3038ba8417f7b3ad7e37980df520b14fccc9f4888df78be2519a
 size 268963141

ckpt-40-percent/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:17d5a3e321b13968af04496b05cee2e429ee248e578f5a3d0ae41b3721da2857
-size 14709
+oid sha256:7dbc8c9973e8ee68a48d381267c6f9df9f7ae9da4f171318dc9c7a0f1150af01
+size 14645

ckpt-40-percent/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:903719f9c6dd8b4ae758b702075c2357a375dcc44c265e022a8de9fb3959abdb
+oid sha256:5fbb20913bae5335d75c036bc67049a0f424c18b248b7ab0fe9f7a3664d090e3
 size 1465

ckpt-40-percent/trainer_state.json

@@ -2,831 +2,809 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.1450381679389313,
+  "epoch": 0.3978494623655914,
   "eval_steps": 500,
-  "global_step": 38,
+  "global_step": 37,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 567.1225036621094,
-      "completions/clipped_ratio": 0.07750000059604645,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1791.0,
-      "completions/mean_length": 568.0449829101562,
-      "completions/mean_terminated_length": 443.7127380371094,
-      "completions/min_length": 31.0,
-      "completions/min_terminated_length": 31.0,
-      "epoch": 0.003816793893129771,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.1118144765496254,
+      "completion_length": 417.31876068115236,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1219.0,
+      "completions/max_terminated_length": 1219.0,
+      "completions/mean_length": 418.3187255859375,
+      "completions/mean_terminated_length": 418.3187255859375,
+      "completions/min_length": 155.0,
+      "completions/min_terminated_length": 155.0,
+      "epoch": 0.010752688172043012,
+      "frac_reward_zero_std": 0.2750000059604645,
+      "grad_norm": 0.1586368829011917,
       "kl": 0.0,
       "learning_rate": 0.0,
       "loss": 0.0,
-      "num_tokens": 359368.0,
-      "reward": 0.3113526701927185,
-      "reward_std": 0.43317943811416626,
-      "rewards/multidomain_reward_func/mean": 0.3113526999950409,
-      "rewards/multidomain_reward_func/std": 1.7456704378128052,
+      "num_tokens": 589935.0,
+      "reward": 0.9395886063575745,
+      "reward_std": 0.6425187587738037,
+      "rewards/multidomain_reward_func/mean": 0.9395885467529297,
+      "rewards/multidomain_reward_func/std": 1.1834609508514404,
       "step": 1
     },
     {
-      "completion_length": 426.60000762939455,
+      "completion_length": 447.3875045776367,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 1055.0,
-      "completions/max_terminated_length": 1055.0,
-      "completions/mean_length": 427.5999755859375,
-      "completions/mean_terminated_length": 427.5999755859375,
-      "completions/min_length": 125.0,
-      "completions/min_terminated_length": 125.0,
-      "epoch": 0.007633587786259542,
-      "frac_reward_zero_std": 0.22499999403953552,
-      "grad_norm": 0.18109840154647827,
-      "kl": 0.0,
-      "learning_rate": 1.25e-06,
+      "completions/max_length": 1311.0,
+      "completions/max_terminated_length": 1311.0,
+      "completions/mean_length": 448.3874816894531,
+      "completions/mean_terminated_length": 448.3874816894531,
+      "completions/min_length": 144.0,
+      "completions/min_terminated_length": 144.0,
+      "epoch": 0.021505376344086023,
+      "frac_reward_zero_std": 0.2874999940395355,
+      "grad_norm": 0.09217020124197006,
+      "kl": 0.0,
+      "learning_rate": 9.999999999999999e-06,
       "loss": 0.0,
-      "num_tokens": 666388.0,
-      "reward": 0.9589062333106995,
-      "reward_std": 0.7145926356315613,
-      "rewards/multidomain_reward_func/mean": 0.9589062333106995,
-      "rewards/multidomain_reward_func/std": 1.3675330877304077,
+      "num_tokens": 1202245.0,
+      "reward": 1.0898984670639038,
+      "reward_std": 0.6939569115638733,
+      "rewards/multidomain_reward_func/mean": 1.0898985862731934,
+      "rewards/multidomain_reward_func/std": 1.1529914140701294,
       "step": 2
     },
     {
-      "completion_length": 398.95250396728517,
+      "completion_length": 444.58375854492186,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 1163.0,
-      "completions/max_terminated_length": 1163.0,
-      "completions/mean_length": 399.9524841308594,
-      "completions/mean_terminated_length": 399.9524841308594,
-      "completions/min_length": 129.0,
-      "completions/min_terminated_length": 129.0,
-      "epoch": 0.011450381679389313,
-      "frac_reward_zero_std": 0.17499999701976776,
-      "grad_norm": 0.1430462747812271,
-      "kl": 0.0,
-      "learning_rate": 2.5e-06,
+      "completions/max_length": 1050.0,
+      "completions/max_terminated_length": 1050.0,
+      "completions/mean_length": 445.583740234375,
+      "completions/mean_terminated_length": 445.583740234375,
+      "completions/min_length": 144.0,
+      "completions/min_terminated_length": 144.0,
+      "epoch": 0.03225806451612903,
+      "frac_reward_zero_std": 0.2874999940395355,
+      "grad_norm": 0.09536850452423096,
+      "kl": 0.0,
+      "learning_rate": 1.9999999999999998e-05,
       "loss": 0.0,
-      "num_tokens": 944729.0,
-      "reward": 0.980177104473114,
-      "reward_std": 0.7184366583824158,
-      "rewards/multidomain_reward_func/mean": 0.980177104473114,
-      "rewards/multidomain_reward_func/std": 1.2778397798538208,
+      "num_tokens": 1822762.0,
+      "reward": 1.2017430067062378,
+      "reward_std": 0.6153996586799622,
+      "rewards/multidomain_reward_func/mean": 1.2017431259155273,
+      "rewards/multidomain_reward_func/std": 1.1850411891937256,
       "step": 3
     },
     {
-      "completion_length": 413.59500885009766,
+      "completion_length": 440.8112548828125,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 1070.0,
-      "completions/max_terminated_length": 1070.0,
-      "completions/mean_length": 414.5950012207031,
-      "completions/mean_terminated_length": 414.5950012207031,
-      "completions/min_length": 20.0,
-      "completions/min_terminated_length": 20.0,
-      "epoch": 0.015267175572519083,
-      "frac_reward_zero_std": 0.14999999105930328,
-      "grad_norm": 0.18651697039604187,
-      "kl": 0.0,
-      "learning_rate": 3.7500000000000005e-06,
+      "completions/max_length": 1121.0,
+      "completions/max_terminated_length": 1121.0,
+      "completions/mean_length": 441.8112487792969,
+      "completions/mean_terminated_length": 441.8112487792969,
+      "completions/min_length": 113.0,
+      "completions/min_terminated_length": 113.0,
+      "epoch": 0.043010752688172046,
+      "frac_reward_zero_std": 0.36249998211860657,
+      "grad_norm": 0.14304442703723907,
+      "kl": 0.0,
+      "learning_rate": 3e-05,
       "loss": -0.0,
-      "num_tokens": 1240667.0,
-      "reward": 0.8180863261222839,
-      "reward_std": 0.7909437417984009,
-      "rewards/multidomain_reward_func/mean": 0.8180863261222839,
-      "rewards/multidomain_reward_func/std": 1.4800753593444824,
+      "num_tokens": 2404141.0,
+      "reward": 0.9961501955986023,
+      "reward_std": 0.5547618865966797,
+      "rewards/multidomain_reward_func/mean": 0.9961501955986023,
+      "rewards/multidomain_reward_func/std": 1.2231403589248657,
       "step": 4
     },
     {
-      "completion_length": 403.8550094604492,
+      "completion_length": 513.616259765625,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 1220.0,
-      "completions/max_terminated_length": 1220.0,
-      "completions/mean_length": 404.85498046875,
-      "completions/mean_terminated_length": 404.85498046875,
-      "completions/min_length": 114.0,
-      "completions/min_terminated_length": 114.0,
-      "epoch": 0.019083969465648856,
-      "frac_reward_zero_std": 0.375,
-      "grad_norm": 0.1282689869403839,
-      "kl": 0.0,
-      "learning_rate": 5e-06,
-      "loss": -0.0,
-      "num_tokens": 1528019.0,
-      "reward": 0.9822691082954407,
-      "reward_std": 0.5212968587875366,
-      "rewards/multidomain_reward_func/mean": 0.9822691082954407,
-      "rewards/multidomain_reward_func/std": 1.1484410762786865,
+      "completions/max_length": 1674.0,
+      "completions/max_terminated_length": 1674.0,
+      "completions/mean_length": 514.6162109375,
+      "completions/mean_terminated_length": 514.6162109375,
+      "completions/min_length": 153.0,
+      "completions/min_terminated_length": 153.0,
+      "epoch": 0.053763440860215055,
+      "frac_reward_zero_std": 0.21249999105930328,
+      "grad_norm": 0.0913516953587532,
+      "kl": 0.0,
+      "learning_rate": 3e-05,
+      "loss": 0.0,
+      "num_tokens": 3070424.0,
+      "reward": 0.9858412146568298,
+      "reward_std": 0.6862795948982239,
+      "rewards/multidomain_reward_func/mean": 0.9858411550521851,
+      "rewards/multidomain_reward_func/std": 1.2488752603530884,
       "step": 5
     },
     {
-      "completion_length": 419.6075042724609,
+      "completion_length": 501.64500885009767,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 1306.0,
-      "completions/max_terminated_length": 1306.0,
-      "completions/mean_length": 420.60748291015625,
-      "completions/mean_terminated_length": 420.60748291015625,
-      "completions/min_length": 123.0,
-      "completions/min_terminated_length": 123.0,
-      "epoch": 0.022900763358778626,
-      "frac_reward_zero_std": 0.32499998807907104,
-      "grad_norm": 0.1297599971294403,
+      "completions/max_length": 1360.0,
+      "completions/max_terminated_length": 1360.0,
+      "completions/mean_length": 502.6449890136719,
+      "completions/mean_terminated_length": 502.6449890136719,
+      "completions/min_length": 128.0,
+      "completions/min_terminated_length": 128.0,
+      "epoch": 0.06451612903225806,
+      "frac_reward_zero_std": 0.29999998211860657,
+      "grad_norm": 0.09110506623983383,
       "kl": 0.0,
-      "learning_rate": 6.25e-06,
+      "learning_rate": 3e-05,
       "loss": 0.0,
-      "num_tokens": 1838792.0,
-      "reward": 1.1234791278839111,
-      "reward_std": 0.5952121019363403,
-      "rewards/multidomain_reward_func/mean": 1.1234791278839111,
-      "rewards/multidomain_reward_func/std": 1.1342366933822632,
+      "num_tokens": 3715110.0,
+      "reward": 1.0306060314178467,
+      "reward_std": 0.6257905960083008,
+      "rewards/multidomain_reward_func/mean": 1.0306060314178467,
+      "rewards/multidomain_reward_func/std": 1.1789323091506958,
       "step": 6
     },
     {
-      "completion_length": 394.5875045776367,
+      "completion_length": 520.2025085449219,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 1297.0,
-      "completions/max_terminated_length": 1297.0,
-      "completions/mean_length": 395.5874938964844,
-      "completions/mean_terminated_length": 395.5874938964844,
-      "completions/min_length": 20.0,
-      "completions/min_terminated_length": 20.0,
-      "epoch": 0.026717557251908396,
-      "frac_reward_zero_std": 0.29999998211860657,
-      "grad_norm": 0.1435064822435379,
-      "kl": 0.0,
-      "learning_rate": 7.500000000000001e-06,
-      "loss": 0.0,
-      "num_tokens": 2127307.0,
-      "reward": 0.9833280444145203,
-      "reward_std": 0.6006773114204407,
-      "rewards/multidomain_reward_func/mean": 0.983328104019165,
-      "rewards/multidomain_reward_func/std": 1.1262794733047485,
+      "completions/max_length": 1468.0,
+      "completions/max_terminated_length": 1468.0,
+      "completions/mean_length": 521.2025146484375,
+      "completions/mean_terminated_length": 521.2025146484375,
+      "completions/min_length": 191.0,
+      "completions/min_terminated_length": 191.0,
+      "epoch": 0.07526881720430108,
+      "frac_reward_zero_std": 0.26249998807907104,
+      "grad_norm": 0.08829142153263092,
+      "kl": 0.0,
+      "learning_rate": 3e-05,
+      "loss": -0.0,
+      "num_tokens": 4384052.0,
+      "reward": 1.1119601726531982,
+      "reward_std": 0.6622768044471741,
+      "rewards/multidomain_reward_func/mean": 1.1119602918624878,
+      "rewards/multidomain_reward_func/std": 1.1772183179855347,
       "step": 7
     },
     {
-      "completion_length": 386.64000701904297,
+      "completion_length": 528.4200134277344,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 1240.0,
-      "completions/max_terminated_length": 1240.0,
-      "completions/mean_length": 387.6399841308594,
-      "completions/mean_terminated_length": 387.6399841308594,
-      "completions/min_length": 99.0,
-      "completions/min_terminated_length": 99.0,
-      "epoch": 0.030534351145038167,
-      "frac_reward_zero_std": 0.22499999403953552,
-      "grad_norm": 0.1412028968334198,
-      "kl": 0.0,
-      "learning_rate": 8.750000000000001e-06,
-      "loss": -0.0,
-      "num_tokens": 2408423.0,
-      "reward": 0.9568594098091125,
-      "reward_std": 0.6804539561271667,
-      "rewards/multidomain_reward_func/mean": 0.9568594098091125,
-      "rewards/multidomain_reward_func/std": 1.3227062225341797,
+      "completions/max_length": 1742.0,
+      "completions/max_terminated_length": 1742.0,
+      "completions/mean_length": 529.4199829101562,
+      "completions/mean_terminated_length": 529.4199829101562,
+      "completions/min_length": 164.0,
+      "completions/min_terminated_length": 164.0,
+      "epoch": 0.08602150537634409,
+      "frac_reward_zero_std": 0.21249999105930328,
+      "grad_norm": 0.09405792504549026,
+      "kl": 0.0,
+      "learning_rate": 3e-05,
+      "loss": 0.0,
+      "num_tokens": 5063198.0,
+      "reward": 0.8146641254425049,
+      "reward_std": 1.0251129865646362,
+      "rewards/multidomain_reward_func/mean": 0.8146640658378601,
+      "rewards/multidomain_reward_func/std": 1.5042277574539185,
       "step": 8
     },
     {
-      "completion_length": 392.76500701904297,
+      "completion_length": 530.1662612915039,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 1124.0,
-      "completions/max_terminated_length": 1124.0,
-      "completions/mean_length": 393.7649841308594,
-      "completions/mean_terminated_length": 393.7649841308594,
-      "completions/min_length": 129.0,
-      "completions/min_terminated_length": 129.0,
-      "epoch": 0.03435114503816794,
-      "frac_reward_zero_std": 0.375,
-      "grad_norm": 0.1385762244462967,
-      "kl": 0.0,
-      "learning_rate": 1e-05,
+      "completions/max_length": 1322.0,
+      "completions/max_terminated_length": 1322.0,
+      "completions/mean_length": 531.166259765625,
+      "completions/mean_terminated_length": 531.166259765625,
+      "completions/min_length": 182.0,
+      "completions/min_terminated_length": 182.0,
+      "epoch": 0.0967741935483871,
+      "frac_reward_zero_std": 0.3125,
+      "grad_norm": 0.08664832264184952,
+      "kl": 0.0,
+      "learning_rate": 3e-05,
       "loss": 0.0,
-      "num_tokens": 2693519.0,
-      "reward": 1.08942711353302,
-      "reward_std": 0.5218599438667297,
-      "rewards/multidomain_reward_func/mean": 1.0894269943237305,
-      "rewards/multidomain_reward_func/std": 1.1226119995117188,
+      "num_tokens": 5745791.0,
+      "reward": 1.105246663093567,
+      "reward_std": 0.5355943441390991,
+      "rewards/multidomain_reward_func/mean": 1.1052465438842773,
+      "rewards/multidomain_reward_func/std": 1.1541340351104736,
       "step": 9
     },
     {
-      "completion_length": 402.0925048828125,
+      "completion_length": 529.9687576293945,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 1010.0,
-      "completions/max_terminated_length": 1010.0,
-      "completions/mean_length": 403.0924987792969,
-      "completions/mean_terminated_length": 403.0924987792969,
-      "completions/min_length": 113.0,
-      "completions/min_terminated_length": 113.0,
-      "epoch": 0.03816793893129771,
-      "frac_reward_zero_std": 0.42499998211860657,
-      "grad_norm": 0.1233278438448906,
-      "kl": 0.0,
-      "learning_rate": 1e-05,
-      "loss": -0.0,
-      "num_tokens": 2988546.0,
-      "reward": 1.0668281316757202,
-      "reward_std": 0.45709723234176636,
-      "rewards/multidomain_reward_func/mean": 1.0668281316757202,
-      "rewards/multidomain_reward_func/std": 1.084045648574829,
+      "completions/max_length": 1527.0,
+      "completions/max_terminated_length": 1527.0,
+      "completions/mean_length": 530.96875,
+      "completions/mean_terminated_length": 530.96875,
+      "completions/min_length": 146.0,
+      "completions/min_terminated_length": 146.0,
+      "epoch": 0.10752688172043011,
+      "frac_reward_zero_std": 0.23749999701976776,
+      "grad_norm": 0.09411562234163284,
+      "kl": 0.0,
+      "learning_rate": 3e-05,
+      "loss": 0.0,
+      "num_tokens": 6448686.0,
+      "reward": 0.5127537846565247,
+      "reward_std": 1.1222161054611206,
+      "rewards/multidomain_reward_func/mean": 0.5127537846565247,
+      "rewards/multidomain_reward_func/std": 1.6059224605560303,
       "step": 10
     },
     {
-      "completion_length": 403.14000701904297,
+      "completion_length": 506.1475082397461,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 1282.0,
-      "completions/max_terminated_length": 1282.0,
-      "completions/mean_length": 404.1399841308594,
-      "completions/mean_terminated_length": 404.1399841308594,
-      "completions/min_length": 85.0,
-      "completions/min_terminated_length": 85.0,
-      "epoch": 0.04198473282442748,
-      "frac_reward_zero_std": 0.19999998807907104,
-      "grad_norm": 0.1325792521238327,
-      "kl": 0.0,
-      "learning_rate": 1e-05,
-      "loss": 0.0,
-      "num_tokens": 3284862.0,
-      "reward": 0.9093489646911621,
-      "reward_std": 0.7637284398078918,
-      "rewards/multidomain_reward_func/mean": 0.9093489646911621,
-      "rewards/multidomain_reward_func/std": 1.334052324295044,
+      "completions/max_length": 1700.0,
+      "completions/max_terminated_length": 1700.0,
+      "completions/mean_length": 507.1474914550781,
+      "completions/mean_terminated_length": 507.1474914550781,
+      "completions/min_length": 101.0,
+      "completions/min_terminated_length": 101.0,
+      "epoch": 0.11827956989247312,
+      "frac_reward_zero_std": 0.3375000059604645,
+      "grad_norm": 0.09998169541358948,
+      "kl": 0.0,
+      "learning_rate": 3e-05,
+      "loss": -0.0,
+      "num_tokens": 7132404.0,
+      "reward": 0.799436628818512,
+      "reward_std": 0.71962970495224,
+      "rewards/multidomain_reward_func/mean": 0.799436628818512,
+      "rewards/multidomain_reward_func/std": 1.4251511096954346,
       "step": 11
     },
     {
-      "completion_length": 417.5600051879883,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 1357.0,
-      "completions/max_terminated_length": 1357.0,
-      "completions/mean_length": 418.55999755859375,
-      "completions/mean_terminated_length": 418.55999755859375,
-      "completions/min_length": 74.0,
-      "completions/min_terminated_length": 74.0,
-      "epoch": 0.04580152671755725,
+      "completion_length": 486.396257019043,
+      "completions/clipped_ratio": 0.0012499999720603228,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1110.0,
+      "completions/mean_length": 487.3949890136719,
+      "completions/mean_terminated_length": 485.4418029785156,
+      "completions/min_length": 165.0,
+      "completions/min_terminated_length": 165.0,
+      "epoch": 0.12903225806451613,
       "frac_reward_zero_std": 0.29999998211860657,
-      "grad_norm": 0.15695820748806,
+      "grad_norm": 0.09192386269569397,
       "kl": 0.0,
-      "learning_rate": 1e-05,
+      "learning_rate": 3e-05,
       "loss": 0.0,
-      "num_tokens": 3594236.0,
-      "reward": 1.1599501371383667,
-      "reward_std": 0.5550302267074585,
-      "rewards/multidomain_reward_func/mean": 1.1599501371383667,
-      "rewards/multidomain_reward_func/std": 1.0896780490875244,
+      "num_tokens": 7759800.0,
+      "reward": 1.1488876342773438,
+      "reward_std": 0.5708559155464172,
+      "rewards/multidomain_reward_func/mean": 1.1488877534866333,
+      "rewards/multidomain_reward_func/std": 1.160998821258545,
       "step": 12
     },
     {
-      "completion_length": 787.7125091552734,
-      "completions/clipped_ratio": 0.20999999344348907,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1967.0,
-      "completions/mean_length": 788.5025024414062,
-      "completions/mean_terminated_length": 453.6993713378906,
-      "completions/min_length": 4.0,
-      "completions/min_terminated_length": 4.0,
-      "epoch": 0.04961832061068702,
-      "frac_reward_zero_std": 0.625,
-      "grad_norm": 0.11839566379785538,
-      "kl": 0.0,
-      "learning_rate": 1e-05,
-      "loss": -0.0,
-      "num_tokens": 4067927.0,
-      "reward": -1.5976874828338623,
-      "reward_std": 0.30139976739883423,
-      "rewards/multidomain_reward_func/mean": -1.5976874828338623,
-      "rewards/multidomain_reward_func/std": 1.9086108207702637,
+      "completion_length": 476.3787567138672,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1302.0,
+      "completions/max_terminated_length": 1302.0,
+      "completions/mean_length": 477.3787536621094,
+      "completions/mean_terminated_length": 477.3787536621094,
+      "completions/min_length": 121.0,
+      "completions/min_terminated_length": 121.0,
+      "epoch": 0.13978494623655913,
+      "frac_reward_zero_std": 0.4124999940395355,
+      "grad_norm": 0.09517515450716019,
+      "kl": 0.0,
+      "learning_rate": 3e-05,
+      "loss": 0.0,
+      "num_tokens": 8412593.0,
+      "reward": 1.0822017192840576,
+      "reward_std": 0.47241905331611633,
+      "rewards/multidomain_reward_func/mean": 1.082201600074768,
+      "rewards/multidomain_reward_func/std": 1.0703767538070679,
       "step": 13
     },
     {
-      "completion_length": 390.23500671386716,
+      "completion_length": 461.2287567138672,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 1288.0,
-      "completions/max_terminated_length": 1288.0,
-      "completions/mean_length": 391.2349853515625,
-      "completions/mean_terminated_length": 391.2349853515625,
-      "completions/min_length": 104.0,
-      "completions/min_terminated_length": 104.0,
-      "epoch": 0.05343511450381679,
-      "frac_reward_zero_std": 0.29999998211860657,
-      "grad_norm": 0.14162577688694,
+      "completions/max_length": 1070.0,
+      "completions/max_terminated_length": 1070.0,
+      "completions/mean_length": 462.2287292480469,
+      "completions/mean_terminated_length": 462.2287292480469,
+      "completions/min_length": 132.0,
+      "completions/min_terminated_length": 132.0,
+      "epoch": 0.15053763440860216,
+      "frac_reward_zero_std": 0.4124999940395355,
+      "grad_norm": 0.08922425657510757,
       "kl": 0.0,
-      "learning_rate": 1e-05,
-      "loss": 0.0,
-      "num_tokens": 4354421.0,
-      "reward": 0.8984739780426025,
-      "reward_std": 0.7180371880531311,
-      "rewards/multidomain_reward_func/mean": 0.8984739780426025,
-      "rewards/multidomain_reward_func/std": 1.231815218925476,
+      "learning_rate": 3e-05,
+      "loss": -0.0,
+      "num_tokens": 9027756.0,
+      "reward": 1.1727960109710693,
+      "reward_std": 0.5148429274559021,
+      "rewards/multidomain_reward_func/mean": 1.1727960109710693,
+      "rewards/multidomain_reward_func/std": 1.1102207899093628,
       "step": 14
     },
     {
-      "completion_length": 383.99000549316406,
-      "completions/clipped_ratio": 0.0024999999441206455,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1316.0,
-      "completions/mean_length": 384.98748779296875,
-      "completions/mean_terminated_length": 380.8195495605469,
-      "completions/min_length": 58.0,
-      "completions/min_terminated_length": 58.0,
-      "epoch": 0.05725190839694656,
-      "frac_reward_zero_std": 0.29999998211860657,
-      "grad_norm": 0.16749094426631927,
+      "completion_length": 447.0050079345703,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1175.0,
+      "completions/max_terminated_length": 1175.0,
+      "completions/mean_length": 448.0050048828125,
+      "completions/mean_terminated_length": 448.0050048828125,
+      "completions/min_length": 125.0,
+      "completions/min_terminated_length": 125.0,
+      "epoch": 0.16129032258064516,
+      "frac_reward_zero_std": 0.25,
+      "grad_norm": 0.09966976195573807,
       "kl": 0.0,
-      "learning_rate": 1e-05,
+      "learning_rate": 3e-05,
       "loss": -0.0,
-      "num_tokens": 4621896.0,
-      "reward": 0.781964123249054,
-      "reward_std": 0.7420740127563477,
-      "rewards/multidomain_reward_func/mean": 0.781964123249054,
-      "rewards/multidomain_reward_func/std": 1.3384652137756348,
+      "num_tokens": 9647350.0,
+      "reward": 1.1358871459960938,
+      "reward_std": 0.6162644028663635,
+      "rewards/multidomain_reward_func/mean": 1.1358871459960938,
+      "rewards/multidomain_reward_func/std": 1.0770015716552734,
       "step": 15
     },
     {
-      "completion_length": 406.53000793457034,
+      "completion_length": 453.9125045776367,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 1692.0,
-      "completions/max_terminated_length": 1692.0,
-      "completions/mean_length": 407.5299987792969,
-      "completions/mean_terminated_length": 407.5299987792969,
-      "completions/min_length": 105.0,
-      "completions/min_terminated_length": 105.0,
-      "epoch": 0.061068702290076333,
-      "frac_reward_zero_std": 0.3499999940395355,
-      "grad_norm": 0.14106978476047516,
-      "kl": 0.0,
-      "learning_rate": 1e-05,
-      "loss": -0.0,
-      "num_tokens": 4914168.0,
-      "reward": 0.8764218688011169,
-      "reward_std": 0.5444123148918152,
-      "rewards/multidomain_reward_func/mean": 0.8764218688011169,
-      "rewards/multidomain_reward_func/std": 1.2579195499420166,
+      "completions/max_length": 1108.0,
+      "completions/max_terminated_length": 1108.0,
+      "completions/mean_length": 454.9124755859375,
+      "completions/mean_terminated_length": 454.9124755859375,
+      "completions/min_length": 118.0,
+      "completions/min_terminated_length": 118.0,
+      "epoch": 0.17204301075268819,
+      "frac_reward_zero_std": 0.26249998807907104,
+      "grad_norm": 0.09356285631656647,
+      "kl": 0.0,
+      "learning_rate": 3e-05,
+      "loss": 0.0,
+      "num_tokens": 10275890.0,
+      "reward": 1.1041463613510132,
+      "reward_std": 0.595342755317688,
+      "rewards/multidomain_reward_func/mean": 1.1041463613510132,
+      "rewards/multidomain_reward_func/std": 1.1405205726623535,
       "step": 16
     },
     {
-      "completion_length": 424.4025085449219,
+      "completion_length": 448.9925048828125,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 1329.0,
-      "completions/max_terminated_length": 1329.0,
-      "completions/mean_length": 425.4024963378906,
-      "completions/mean_terminated_length": 425.4024963378906,
-      "completions/min_length": 107.0,
-      "completions/min_terminated_length": 107.0,
-      "epoch": 0.0648854961832061,
+      "completions/max_length": 1191.0,
+      "completions/max_terminated_length": 1191.0,
+      "completions/mean_length": 449.99249267578125,
+      "completions/mean_terminated_length": 449.99249267578125,
+      "completions/min_length": 142.0,
+      "completions/min_terminated_length": 142.0,
+      "epoch": 0.1827956989247312,
       "frac_reward_zero_std": 0.22499999403953552,
-      "grad_norm": 0.15478958189487457,
+      "grad_norm": 0.10382834821939468,
       "kl": 0.0,
-      "learning_rate": 1e-05,
-      "loss": -0.0,
-      "num_tokens": 5222629.0,
-      "reward": 0.9992671012878418,
-      "reward_std": 0.7444822788238525,
-      "rewards/multidomain_reward_func/mean": 0.9992671012878418,
-      "rewards/multidomain_reward_func/std": 1.1914256811141968,
+      "learning_rate": 3e-05,
+      "loss": 0.0,
+      "num_tokens": 10890984.0,
+      "reward": 1.1085461378097534,
+      "reward_std": 0.6585391163825989,
+      "rewards/multidomain_reward_func/mean": 1.1085461378097534,
+      "rewards/multidomain_reward_func/std": 1.2252981662750244,
       "step": 17
     },
     {
-      "completion_length": 395.4175079345703,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 1124.0,
-      "completions/max_terminated_length": 1124.0,
-      "completions/mean_length": 396.41748046875,
-      "completions/mean_terminated_length": 396.41748046875,
-      "completions/min_length": 127.0,
-      "completions/min_terminated_length": 127.0,
-      "epoch": 0.06870229007633588,
-      "frac_reward_zero_std": 0.25,
-      "grad_norm": 0.14686347544193268,
+      "completion_length": 451.19625701904295,
+      "completions/clipped_ratio": 0.0037499999161809683,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1073.0,
+      "completions/mean_length": 452.1925048828125,
+      "completions/mean_terminated_length": 446.1856994628906,
+      "completions/min_length": 106.0,
+      "completions/min_terminated_length": 106.0,
+      "epoch": 0.1935483870967742,
+      "frac_reward_zero_std": 0.32499998807907104,
+      "grad_norm": 0.10299306362867355,
       "kl": 0.0,
-      "learning_rate": 1e-05,
-      "loss": 0.0,
-      "num_tokens": 5501186.0,
-      "reward": 1.1373611688613892,
-      "reward_std": 0.5914338827133179,
-      "rewards/multidomain_reward_func/mean": 1.1373611688613892,
-      "rewards/multidomain_reward_func/std": 1.1266436576843262,
+      "learning_rate": 3e-05,
+      "loss": -0.0,
+      "num_tokens": 11513098.0,
+      "reward": 0.9830173850059509,
+      "reward_std": 0.5063682198524475,
+      "rewards/multidomain_reward_func/mean": 0.9830173254013062,
+      "rewards/multidomain_reward_func/std": 1.2036722898483276,
       "step": 18
     },
     {
-      "completion_length": 423.48500671386716,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 1373.0,
-      "completions/max_terminated_length": 1373.0,
-      "completions/mean_length": 424.4849853515625,
-      "completions/mean_terminated_length": 424.4849853515625,
-      "completions/min_length": 123.0,
-      "completions/min_terminated_length": 123.0,
-      "epoch": 0.07251908396946564,
+      "completion_length": 474.92625732421874,
+      "completions/clipped_ratio": 0.0012499999720603228,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1312.0,
+      "completions/mean_length": 475.92498779296875,
+      "completions/mean_terminated_length": 473.95745849609375,
+      "completions/min_length": 129.0,
+      "completions/min_terminated_length": 129.0,
+      "epoch": 0.20430107526881722,
       "frac_reward_zero_std": 0.29999998211860657,
-      "grad_norm": 0.13952139019966125,
-      "kl": 0.0,
-      "learning_rate": 1e-05,
-      "loss": -0.0001,
-      "num_tokens": 5795160.0,
-      "reward": 0.9805884957313538,
-      "reward_std": 0.6606646776199341,
-      "rewards/multidomain_reward_func/mean": 0.9805885553359985,
-      "rewards/multidomain_reward_func/std": 1.3177053928375244,
+      "grad_norm": 0.09433634579181671,
+      "kl": 0.0,
+      "learning_rate": 3e-05,
+      "loss": -0.0,
+      "num_tokens": 12184568.0,
+      "reward": 1.0752404928207397,
+      "reward_std": 0.5778380632400513,
+      "rewards/multidomain_reward_func/mean": 1.0752404928207397,
+      "rewards/multidomain_reward_func/std": 1.1373811960220337,
       "step": 19
     },
     {
-      "completion_length": 399.9825042724609,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 1349.0,
-      "completions/max_terminated_length": 1349.0,
-      "completions/mean_length": 400.98248291015625,
-      "completions/mean_terminated_length": 400.98248291015625,
-      "completions/min_length": 106.0,
-      "completions/min_terminated_length": 106.0,
-      "epoch": 0.07633587786259542,
-      "frac_reward_zero_std": 0.29999998211860657,
-      "grad_norm": 0.1434020698070526,
-      "kl": 0.0,
-      "learning_rate": 1e-05,
+      "completion_length": 452.972509765625,
+      "completions/clipped_ratio": 0.0012499999720603228,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1264.0,
+      "completions/mean_length": 453.97125244140625,
+      "completions/mean_terminated_length": 451.9762268066406,
+      "completions/min_length": 102.0,
+      "completions/min_terminated_length": 102.0,
+      "epoch": 0.21505376344086022,
+      "frac_reward_zero_std": 0.23749999701976776,
+      "grad_norm": 0.09762763977050781,
+      "kl": 0.0,
+      "learning_rate": 3e-05,
       "loss": -0.0,
-      "num_tokens": 6067803.0,
-      "reward": 1.0183696746826172,
-      "reward_std": 0.6196312308311462,
-      "rewards/multidomain_reward_func/mean": 1.0183697938919067,
-      "rewards/multidomain_reward_func/std": 1.0895596742630005,
+      "num_tokens": 12820175.0,
+      "reward": 0.7925729155540466,
+      "reward_std": 0.8282187581062317,
+      "rewards/multidomain_reward_func/mean": 0.7925729155540466,
+      "rewards/multidomain_reward_func/std": 1.3876315355300903,
       "step": 20
     },
     {
-      "completion_length": 389.4550079345703,
+      "completion_length": 430.3100082397461,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 1196.0,
-      "completions/max_terminated_length": 1196.0,
-      "completions/mean_length": 390.4549865722656,
-      "completions/mean_terminated_length": 390.4549865722656,
-      "completions/min_length": 112.0,
-      "completions/min_terminated_length": 112.0,
-      "epoch": 0.08015267175572519,
-      "frac_reward_zero_std": 0.2750000059604645,
-      "grad_norm": 0.14288610219955444,
+      "completions/max_length": 974.0,
+      "completions/max_terminated_length": 974.0,
+      "completions/mean_length": 431.30999755859375,
+      "completions/mean_terminated_length": 431.30999755859375,
+      "completions/min_length": 136.0,
+      "completions/min_terminated_length": 136.0,
+      "epoch": 0.22580645161290322,
+      "frac_reward_zero_std": 0.32499998807907104,
+      "grad_norm": 0.12044712156057358,
       "kl": 0.0,
-      "learning_rate": 1e-05,
+      "learning_rate": 3e-05,
       "loss": 0.0,
-      "num_tokens": 6343805.0,
-      "reward": 1.1138594150543213,
-      "reward_std": 0.696417510509491,
-      "rewards/multidomain_reward_func/mean": 1.1138594150543213,
-      "rewards/multidomain_reward_func/std": 1.0503205060958862,
+      "num_tokens": 13426793.0,
+      "reward": 1.1226346492767334,
+      "reward_std": 0.514378011226654,
+      "rewards/multidomain_reward_func/mean": 1.122634768486023,
+      "rewards/multidomain_reward_func/std": 1.1829208135604858,
       "step": 21
     },
     {
-      "completion_length": 417.29000701904295,
-      "completions/clipped_ratio": 0.004999999888241291,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1233.0,
-      "completions/mean_length": 418.2850036621094,
-      "completions/mean_terminated_length": 410.095458984375,
-      "completions/min_length": 112.0,
-      "completions/min_terminated_length": 112.0,
-      "epoch": 0.08396946564885496,
-      "frac_reward_zero_std": 0.2750000059604645,
-      "grad_norm": 0.13009311258792877,
-      "kl": 0.0,
-      "learning_rate": 1e-05,
-      "loss": -0.0,
-      "num_tokens": 6627459.0,
-      "reward": 1.0032275915145874,
-      "reward_std": 0.6355298757553101,
-      "rewards/multidomain_reward_func/mean": 1.003227710723877,
-      "rewards/multidomain_reward_func/std": 1.3018079996109009,
+      "completion_length": 412.8612548828125,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1069.0,
+      "completions/max_terminated_length": 1069.0,
+      "completions/mean_length": 413.8612365722656,
+      "completions/mean_terminated_length": 413.8612365722656,
+      "completions/min_length": 117.0,
+      "completions/min_terminated_length": 117.0,
+      "epoch": 0.23655913978494625,
+      "frac_reward_zero_std": 0.21249999105930328,
+      "grad_norm": 0.10490359365940094,
+      "kl": 0.0,
+      "learning_rate": 3e-05,
+      "loss": 0.0,
+      "num_tokens": 14010292.0,
+      "reward": 1.0972095727920532,
+      "reward_std": 0.6835877895355225,
+      "rewards/multidomain_reward_func/mean": 1.0972095727920532,
+      "rewards/multidomain_reward_func/std": 1.1799525022506714,
       "step": 22
     },
     {
-      "completion_length": 435.9750045776367,
-      "completions/clipped_ratio": 0.0024999999441206455,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1614.0,
-      "completions/mean_length": 436.9725036621094,
-      "completions/mean_terminated_length": 432.9348449707031,
-      "completions/min_length": 60.0,
-      "completions/min_terminated_length": 60.0,
-      "epoch": 0.08778625954198473,
-      "frac_reward_zero_std": 0.32499998807907104,
-      "grad_norm": 0.16062505543231964,
+      "completion_length": 398.9125061035156,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1085.0,
+      "completions/max_terminated_length": 1085.0,
+      "completions/mean_length": 399.9125061035156,
+      "completions/mean_terminated_length": 399.9125061035156,
+      "completions/min_length": 118.0,
+      "completions/min_terminated_length": 118.0,
+      "epoch": 0.24731182795698925,
+      "frac_reward_zero_std": 0.29999998211860657,
+      "grad_norm": 0.10093961656093597,
       "kl": 0.0,
-      "learning_rate": 1e-05,
-      "loss": -0.0,
-      "num_tokens": 6931618.0,
-      "reward": 0.8509132266044617,
-      "reward_std": 0.7453229427337646,
-      "rewards/multidomain_reward_func/mean": 0.8509131669998169,
-      "rewards/multidomain_reward_func/std": 1.4180208444595337,
+      "learning_rate": 3e-05,
+      "loss": 0.0,
+      "num_tokens": 14575372.0,
+      "reward": 1.0390172004699707,
+      "reward_std": 0.6097813248634338,
+      "rewards/multidomain_reward_func/mean": 1.0390172004699707,
+      "rewards/multidomain_reward_func/std": 1.273277759552002,
       "step": 23
     },
     {
-      "completion_length": 434.4600067138672,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 1097.0,
-      "completions/max_terminated_length": 1097.0,
-      "completions/mean_length": 435.4599914550781,
-      "completions/mean_terminated_length": 435.4599914550781,
-      "completions/min_length": 131.0,
-      "completions/min_terminated_length": 131.0,
-      "epoch": 0.0916030534351145,
-      "frac_reward_zero_std": 0.3499999940395355,
-      "grad_norm": 0.11939506977796555,
-      "kl": 0.0,
-      "learning_rate": 1e-05,
+      "completion_length": 436.89375457763674,
+      "completions/clipped_ratio": 0.0012499999720603228,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1535.0,
+      "completions/mean_length": 437.8924865722656,
+      "completions/mean_terminated_length": 435.8773498535156,
+      "completions/min_length": 55.0,
+      "completions/min_terminated_length": 55.0,
+      "epoch": 0.25806451612903225,
+      "frac_reward_zero_std": 0.3125,
+      "grad_norm": 0.10864879190921783,
+      "kl": 0.0,
+      "learning_rate": 3e-05,
       "loss": 0.0,
-      "num_tokens": 7230562.0,
-      "reward": 1.0885521173477173,
-      "reward_std": 0.5634962320327759,
-      "rewards/multidomain_reward_func/mean": 1.0885521173477173,
-      "rewards/multidomain_reward_func/std": 1.1546189785003662,
+      "num_tokens": 15207466.0,
+      "reward": 0.8946269154548645,
+      "reward_std": 0.6158351898193359,
+      "rewards/multidomain_reward_func/mean": 0.8946268558502197,
+      "rewards/multidomain_reward_func/std": 1.3325064182281494,
       "step": 24
     },
     {
-      "completion_length": 403.5975036621094,
+      "completion_length": 429.6425079345703,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 1309.0,
-      "completions/max_terminated_length": 1309.0,
-      "completions/mean_length": 404.5975036621094,
-      "completions/mean_terminated_length": 404.5975036621094,
-      "completions/min_length": 132.0,
-      "completions/min_terminated_length": 132.0,
-      "epoch": 0.09541984732824428,
-      "frac_reward_zero_std": 0.19999998807907104,
-      "grad_norm": 0.13339835405349731,
+      "completions/max_length": 1189.0,
+      "completions/max_terminated_length": 1189.0,
+      "completions/mean_length": 430.6424865722656,
+      "completions/mean_terminated_length": 430.6424865722656,
+      "completions/min_length": 123.0,
+      "completions/min_terminated_length": 123.0,
+      "epoch": 0.26881720430107525,
+      "frac_reward_zero_std": 0.22499999403953552,
+      "grad_norm": 0.09544331580400467,
       "kl": 0.0,
-      "learning_rate": 1e-05,
+      "learning_rate": 3e-05,
       "loss": 0.0,
-      "num_tokens": 7509601.0,
-      "reward": 1.0083370208740234,
-      "reward_std": 0.7597755193710327,
-      "rewards/multidomain_reward_func/mean": 1.008337140083313,
-      "rewards/multidomain_reward_func/std": 1.158740520477295,
+      "num_tokens": 15806970.0,
+      "reward": 1.0628045797348022,
+      "reward_std": 0.6665589213371277,
+      "rewards/multidomain_reward_func/mean": 1.0628045797348022,
+      "rewards/multidomain_reward_func/std": 1.2315187454223633,
       "step": 25
     },
     {
-      "completion_length": 432.5300094604492,
-      "completions/clipped_ratio": 0.01249999925494194,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1090.0,
-      "completions/mean_length": 433.5174865722656,
-      "completions/mean_terminated_length": 413.0810241699219,
-      "completions/min_length": 76.0,
-      "completions/min_terminated_length": 76.0,
-      "epoch": 0.09923664122137404,
-      "frac_reward_zero_std": 0.2750000059604645,
-      "grad_norm": 0.21664012968540192,
-      "kl": 0.0,
-      "learning_rate": 1e-05,
+      "completion_length": 431.210009765625,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1029.0,
+      "completions/max_terminated_length": 1029.0,
+      "completions/mean_length": 432.2099914550781,
+      "completions/mean_terminated_length": 432.2099914550781,
+      "completions/min_length": 115.0,
+      "completions/min_terminated_length": 115.0,
+      "epoch": 0.27956989247311825,
+      "frac_reward_zero_std": 0.2874999940395355,
+      "grad_norm": 0.17953357100486755,
+      "kl": 0.0,
+      "learning_rate": 3e-05,
       "loss": 0.0,
-      "num_tokens": 7808048.0,
-      "reward": 0.5465364456176758,
-      "reward_std": 0.7330557703971863,
-      "rewards/multidomain_reward_func/mean": 0.5465364456176758,
-      "rewards/multidomain_reward_func/std": 1.6296391487121582,
+      "num_tokens": 16390478.0,
+      "reward": 1.0917786359786987,
+      "reward_std": 0.5078949332237244,
+      "rewards/multidomain_reward_func/mean": 1.0917787551879883,
+      "rewards/multidomain_reward_func/std": 1.2241995334625244,
       "step": 26
     },
     {
-      "completion_length": 470.5725036621094,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 1447.0,
-      "completions/max_terminated_length": 1447.0,
-      "completions/mean_length": 471.5724792480469,
-      "completions/mean_terminated_length": 471.5724792480469,
-      "completions/min_length": 67.0,
-      "completions/min_terminated_length": 67.0,
-      "epoch": 0.10305343511450382,
-      "frac_reward_zero_std": 0.32499998807907104,
-      "grad_norm": 0.14879898726940155,
+      "completion_length": 469.45375061035156,
+      "completions/clipped_ratio": 0.0037499999161809683,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1498.0,
+      "completions/mean_length": 470.4499816894531,
+      "completions/mean_terminated_length": 464.51190185546875,
+      "completions/min_length": 105.0,
+      "completions/min_terminated_length": 105.0,
+      "epoch": 0.2903225806451613,
+      "frac_reward_zero_std": 0.29999998211860657,
+      "grad_norm": 0.15617702901363373,
       "kl": 0.0,
-      "learning_rate": 1e-05,
+      "learning_rate": 3e-05,
       "loss": 0.0,
-      "num_tokens": 8137947.0,
-      "reward": 1.0213390588760376,
-      "reward_std": 0.6532722115516663,
-      "rewards/multidomain_reward_func/mean": 1.0213390588760376,
-      "rewards/multidomain_reward_func/std": 1.0712261199951172,
+      "num_tokens": 17049938.0,
+      "reward": 0.8478901386260986,
+      "reward_std": 0.5746238827705383,
+      "rewards/multidomain_reward_func/mean": 0.8478901386260986,
+      "rewards/multidomain_reward_func/std": 1.3466750383377075,
       "step": 27
     },
     {
-      "completion_length": 497.59501190185546,
-      "completions/clipped_ratio": 0.01249999925494194,
+      "completion_length": 469.49500427246096,
+      "completions/clipped_ratio": 0.0012499999720603228,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1324.0,
-      "completions/mean_length": 498.5824890136719,
-      "completions/mean_terminated_length": 478.9696350097656,
-      "completions/min_length": 101.0,
-      "completions/min_terminated_length": 101.0,
-      "epoch": 0.10687022900763359,
-      "frac_reward_zero_std": 0.2750000059604645,
-      "grad_norm": 0.13745573163032532,
+      "completions/max_terminated_length": 1457.0,
+      "completions/mean_length": 470.4937438964844,
+      "completions/mean_terminated_length": 468.5194091796875,
+      "completions/min_length": 120.0,
+      "completions/min_terminated_length": 120.0,
+      "epoch": 0.3010752688172043,
+      "frac_reward_zero_std": 0.22499999403953552,
+      "grad_norm": 0.11228151619434357,
       "kl": 0.0,
-      "learning_rate": 1e-05,
+      "learning_rate": 3e-05,
       "loss": 0.0,
-      "num_tokens": 8486510.0,
-      "reward": 0.6380816102027893,
-      "reward_std": 0.6292651891708374,
-      "rewards/multidomain_reward_func/mean": 0.6380816102027893,
-      "rewards/multidomain_reward_func/std": 1.474389910697937,
+      "num_tokens": 17697093.0,
+      "reward": 1.025890588760376,
+      "reward_std": 0.6576307415962219,
+      "rewards/multidomain_reward_func/mean": 1.0258907079696655,
+      "rewards/multidomain_reward_func/std": 1.2718981504440308,
       "step": 28
     },
     {
-      "completion_length": 475.4775100708008,
-      "completions/clipped_ratio": 0.009999999776482582,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1549.0,
-      "completions/mean_length": 476.4674987792969,
-      "completions/mean_terminated_length": 460.59344482421875,
-      "completions/min_length": 67.0,
-      "completions/min_terminated_length": 67.0,
-      "epoch": 0.11068702290076336,
-      "frac_reward_zero_std": 0.2750000059604645,
-      "grad_norm": 0.18127009272575378,
-      "kl": 0.0,
-      "learning_rate": 1e-05,
+      "completion_length": 454.85375823974607,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 1260.0,
+      "completions/max_terminated_length": 1260.0,
+      "completions/mean_length": 455.8537292480469,
+      "completions/mean_terminated_length": 455.8537292480469,
+      "completions/min_length": 118.0,
+      "completions/min_terminated_length": 118.0,
+      "epoch": 0.3118279569892473,
+      "frac_reward_zero_std": 0.23749999701976776,
+      "grad_norm": 0.08834511786699295,
+      "kl": 0.0,
+      "learning_rate": 3e-05,
       "loss": 0.0,
-      "num_tokens": 8811727.0,
-      "reward": 0.3718194365501404,
-      "reward_std": 0.8439445495605469,
-      "rewards/multidomain_reward_func/mean": 0.37181946635246277,
-      "rewards/multidomain_reward_func/std": 1.7001434564590454,
+      "num_tokens": 18325316.0,
+      "reward": 1.1648790836334229,
+      "reward_std": 0.6541204452514648,
+      "rewards/multidomain_reward_func/mean": 1.1648792028427124,
+      "rewards/multidomain_reward_func/std": 1.1322048902511597,
       "step": 29
     },
     {
-      "completion_length": 482.22750701904295,
+      "completion_length": 437.74625701904296,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 1534.0,
-      "completions/max_terminated_length": 1534.0,
-      "completions/mean_length": 483.22747802734375,
-      "completions/mean_terminated_length": 483.22747802734375,
-      "completions/min_length": 126.0,
-      "completions/min_terminated_length": 126.0,
-      "epoch": 0.11450381679389313,
-      "frac_reward_zero_std": 0.32499998807907104,
-      "grad_norm": 0.11341980844736099,
+      "completions/max_length": 1771.0,
+      "completions/max_terminated_length": 1771.0,
+      "completions/mean_length": 438.7462463378906,
+      "completions/mean_terminated_length": 438.7462463378906,
+      "completions/min_length": 113.0,
+      "completions/min_terminated_length": 113.0,
+      "epoch": 0.3225806451612903,
+      "frac_reward_zero_std": 0.23749999701976776,
+      "grad_norm": 0.10401499271392822,
       "kl": 0.0,
-      "learning_rate": 1e-05,
+      "learning_rate": 3e-05,
       "loss": 0.0,
-      "num_tokens": 9151548.0,
-      "reward": 0.9484896063804626,
-      "reward_std": 0.5695937871932983,
-      "rewards/multidomain_reward_func/mean": 0.9484896063804626,
-      "rewards/multidomain_reward_func/std": 1.1334481239318848,
+      "num_tokens": 18941413.0,
+      "reward": 1.0463263988494873,
+      "reward_std": 0.6855629682540894,
+      "rewards/multidomain_reward_func/mean": 1.0463263988494873,
+      "rewards/multidomain_reward_func/std": 1.205276608467102,
       "step": 30
     },
     {
-      "completion_length": 420.1550048828125,
+      "completion_length": 447.941259765625,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 1361.0,
-      "completions/max_terminated_length": 1361.0,
-      "completions/mean_length": 421.1549987792969,
-      "completions/mean_terminated_length": 421.1549987792969,
-      "completions/min_length": 142.0,
-      "completions/min_terminated_length": 142.0,
-      "epoch": 0.1183206106870229,
-      "frac_reward_zero_std": 0.22499999403953552,
-      "grad_norm": 0.13916772603988647,
+      "completions/max_length": 2021.0,
+      "completions/max_terminated_length": 2021.0,
+      "completions/mean_length": 448.9412536621094,
+      "completions/mean_terminated_length": 448.9412536621094,
+      "completions/min_length": 120.0,
+      "completions/min_terminated_length": 120.0,
+      "epoch": 0.3333333333333333,
+      "frac_reward_zero_std": 0.25,
+      "grad_norm": 0.09366130828857422,
       "kl": 0.0,
-      "learning_rate": 1e-05,
+      "learning_rate": 3e-05,
       "loss": -0.0,
-      "num_tokens": 9432380.0,
-      "reward": 0.8056875467300415,
-      "reward_std": 0.7789397835731506,
-      "rewards/multidomain_reward_func/mean": 0.8056875467300415,
-      "rewards/multidomain_reward_func/std": 1.314136028289795,
+      "num_tokens": 19542566.0,
+      "reward": 1.0071501731872559,
+      "reward_std": 0.676040768623352,
+      "rewards/multidomain_reward_func/mean": 1.0071501731872559,
+      "rewards/multidomain_reward_func/std": 1.1646482944488525,
       "step": 31
     },
     {
-      "completion_length": 471.6550033569336,
-      "completions/clipped_ratio": 0.0024999999441206455,
+      "completion_length": 470.27625427246096,
+      "completions/clipped_ratio": 0.0037499999161809683,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1544.0,
-      "completions/mean_length": 472.6524963378906,
-      "completions/mean_terminated_length": 468.7042541503906,
-      "completions/min_length": 30.0,
-      "completions/min_terminated_length": 30.0,
-      "epoch": 0.12213740458015267,
-      "frac_reward_zero_std": 0.25,
-      "grad_norm": 0.1645308881998062,
+      "completions/max_terminated_length": 1493.0,
+      "completions/mean_length": 471.2724914550781,
+      "completions/mean_terminated_length": 465.3375244140625,
+      "completions/min_length": 123.0,
+      "completions/min_terminated_length": 123.0,
+      "epoch": 0.34408602150537637,
+      "frac_reward_zero_std": 0.3125,
+      "grad_norm": 0.09432131797075272,
       "kl": 0.0,
-      "learning_rate": 1e-05,
+      "learning_rate": 3e-05,
       "loss": -0.0,
-      "num_tokens": 9755281.0,
-      "reward": 0.6055446267127991,
-      "reward_std": 0.7402985692024231,
-      "rewards/multidomain_reward_func/mean": 0.6055446863174438,
-      "rewards/multidomain_reward_func/std": 1.5184121131896973,
+      "num_tokens": 20172294.0,
+      "reward": 0.02558271773159504,
+      "reward_std": 1.2057312726974487,
+      "rewards/multidomain_reward_func/mean": 0.02558271400630474,
+      "rewards/multidomain_reward_func/std": 1.7726387977600098,
       "step": 32
     },
     {
-      "completion_length": 459.3675048828125,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 1138.0,
-      "completions/max_terminated_length": 1138.0,
-      "completions/mean_length": 460.36749267578125,
-      "completions/mean_terminated_length": 460.36749267578125,
-      "completions/min_length": 125.0,
-      "completions/min_terminated_length": 125.0,
-      "epoch": 0.12595419847328243,
-      "frac_reward_zero_std": 0.14999999105930328,
-      "grad_norm": 0.1528780460357666,
+      "completion_length": 419.5925033569336,
+      "completions/clipped_ratio": 0.0012499999720603228,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1442.0,
+      "completions/mean_length": 420.59124755859375,
+      "completions/mean_terminated_length": 418.554443359375,
+      "completions/min_length": 111.0,
+      "completions/min_terminated_length": 111.0,
+      "epoch": 0.3548387096774194,
+      "frac_reward_zero_std": 0.3499999940395355,
+      "grad_norm": 0.09137614816427231,
       "kl": 0.0,
-      "learning_rate": 1e-05,
+      "learning_rate": 3e-05,
       "loss": 0.0,
-      "num_tokens": 10076308.0,
-      "reward": 0.9635868072509766,
-      "reward_std": 0.8727088570594788,
-      "rewards/multidomain_reward_func/mean": 0.9635868072509766,
-      "rewards/multidomain_reward_func/std": 1.286659598350525,
+      "num_tokens": 20744097.0,
+      "reward": 1.1542513370513916,
+      "reward_std": 0.5509113073348999,
+      "rewards/multidomain_reward_func/mean": 1.1542514562606812,
+      "rewards/multidomain_reward_func/std": 0.9433695077896118,
       "step": 33
     },
     {
-      "completion_length": 413.8675033569336,
+      "completion_length": 434.2000076293945,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 1067.0,
-      "completions/max_terminated_length": 1067.0,
-      "completions/mean_length": 414.86749267578125,
-      "completions/mean_terminated_length": 414.86749267578125,
-      "completions/min_length": 134.0,
-      "completions/min_terminated_length": 134.0,
-      "epoch": 0.1297709923664122,
-      "frac_reward_zero_std": 0.29999998211860657,
-      "grad_norm": 0.14397776126861572,
-      "kl": 0.0,
-      "learning_rate": 1e-05,
-      "loss": -0.0,
-      "num_tokens": 10360115.0,
-      "reward": 1.0364062786102295,
-      "reward_std": 0.690801203250885,
-      "rewards/multidomain_reward_func/mean": 1.0364062786102295,
-      "rewards/multidomain_reward_func/std": 1.1974419355392456,
+      "completions/max_length": 1262.0,
+      "completions/max_terminated_length": 1262.0,
+      "completions/mean_length": 435.1999816894531,
+      "completions/mean_terminated_length": 435.1999816894531,
+      "completions/min_length": 120.0,
+      "completions/min_terminated_length": 120.0,
+      "epoch": 0.3655913978494624,
+      "frac_reward_zero_std": 0.2874999940395355,
+      "grad_norm": 0.08975847065448761,
+      "kl": 0.0,
+      "learning_rate": 3e-05,
+      "loss": 0.0,
+      "num_tokens": 21349127.0,
+      "reward": 1.0992751121520996,
+      "reward_std": 0.532092809677124,
+      "rewards/multidomain_reward_func/mean": 1.0992752313613892,
+      "rewards/multidomain_reward_func/std": 1.0367480516433716,
       "step": 34
     },
     {
-      "completion_length": 641.300015258789,
-      "completions/clipped_ratio": 0.08749999850988388,
+      "completion_length": 482.0337600708008,
+      "completions/clipped_ratio": 0.0012499999720603228,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 2008.0,
-      "completions/mean_length": 642.2124633789062,
-      "completions/mean_terminated_length": 507.4109802246094,
-      "completions/min_length": 89.0,
-      "completions/min_terminated_length": 89.0,
-      "epoch": 0.13358778625954199,
-      "frac_reward_zero_std": 0.3999999761581421,
-      "grad_norm": 0.11470023542642593,
-      "kl": 0.0,
-      "learning_rate": 1e-05,
+      "completions/max_terminated_length": 2013.0,
+      "completions/mean_length": 483.0325012207031,
+      "completions/mean_terminated_length": 481.0738525390625,
+      "completions/min_length": 118.0,
+      "completions/min_terminated_length": 118.0,
+      "epoch": 0.3763440860215054,
+      "frac_reward_zero_std": 0.25,
+      "grad_norm": 0.09360182285308838,
+      "kl": 0.0,
+      "learning_rate": 3e-05,
       "loss": 0.0,
-      "num_tokens": 10754530.0,
-      "reward": -0.2579292953014374,
-      "reward_std": 0.8382142186164856,
-      "rewards/multidomain_reward_func/mean": -0.2579292953014374,
-      "rewards/multidomain_reward_func/std": 2.043348789215088,
+      "num_tokens": 22033873.0,
+      "reward": 1.0292495489120483,
+      "reward_std": 0.6627224087715149,
+      "rewards/multidomain_reward_func/mean": 1.0292495489120483,
+      "rewards/multidomain_reward_func/std": 1.2340757846832275,
       "step": 35
     },
     {
-      "completion_length": 434.5075088500977,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 1103.0,
-      "completions/max_terminated_length": 1103.0,
-      "completions/mean_length": 435.5074768066406,
-      "completions/mean_terminated_length": 435.5074768066406,
-      "completions/min_length": 124.0,
-      "completions/min_terminated_length": 124.0,
-      "epoch": 0.13740458015267176,
-      "frac_reward_zero_std": 0.3999999761581421,
-      "grad_norm": 0.1501513123512268,
-      "kl": 0.0,
-      "learning_rate": 1e-05,
+      "completion_length": 1182.0312561035157,
+      "completions/clipped_ratio": 0.4074999988079071,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1497.0,
+      "completions/mean_length": 1182.623779296875,
+      "completions/mean_terminated_length": 587.4493408203125,
+      "completions/min_length": 6.0,
+      "completions/min_terminated_length": 6.0,
+      "epoch": 0.3870967741935484,
+      "frac_reward_zero_std": 0.675000011920929,
+      "grad_norm": 0.05318624526262283,
+      "kl": 0.0,
+      "learning_rate": 3e-05,
       "loss": 0.0,
-      "num_tokens": 11054083.0,
-      "reward": 1.0775103569030762,
-      "reward_std": 0.6445066928863525,
-      "rewards/multidomain_reward_func/mean": 1.0775104761123657,
-      "rewards/multidomain_reward_func/std": 1.1158509254455566,
+      "num_tokens": 23279772.0,
+      "reward": -0.7171798348426819,
+      "reward_std": 0.26930734515190125,
+      "rewards/multidomain_reward_func/mean": -0.7171798944473267,
+      "rewards/multidomain_reward_func/std": 2.1405746936798096,
       "step": 36
-    },
-    {
-      "completion_length": 427.5175033569336,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 1242.0,
-      "completions/max_terminated_length": 1242.0,
-      "completions/mean_length": 428.5174865722656,
-      "completions/mean_terminated_length": 428.5174865722656,
-      "completions/min_length": 116.0,
-      "completions/min_terminated_length": 116.0,
-      "epoch": 0.14122137404580154,
-      "frac_reward_zero_std": 0.2750000059604645,
-      "grad_norm": 0.1382102072238922,
-      "kl": 0.0,
-      "learning_rate": 1e-05,
-      "loss": -0.0,
-      "num_tokens": 11357730.0,
-      "reward": 0.934586763381958,
-      "reward_std": 0.6311339139938354,
-      "rewards/multidomain_reward_func/mean": 0.934586763381958,
-      "rewards/multidomain_reward_func/std": 1.2222440242767334,
-      "step": 37
     }
   ],
   "logging_steps": 1,
-  "max_steps": 262,
-  "num_input_tokens_seen": 11657878,
+  "max_steps": 93,
+  "num_input_tokens_seen": 23871076,
   "num_train_epochs": 1,
   "save_steps": 250,
   "stateful_callbacks": {
@@ -842,7 +820,7 @@
     }
   },
   "total_flos": 0.0,
-  "train_batch_size": 20,
+  "train_batch_size": 40,
   "trial_name": null,
   "trial_params": null
 }

ckpt-40-percent/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c8b716515b4f5eb2059f89ec3fda03b45958ba6ea937a4d3de40351b00d5d1f3
+oid sha256:b48077003af5f11ced05cb103bb98c595f8752173ade63e67e1251ccaecb2486
 size 7505