diff --git a/.gitattributes b/.gitattributes index 56327c10364fd7eb0b07b1e4e1af0ce1bc1e3fb1..99cce343792cbb7ccc72921e4f91f8b0a2f500f2 100644 --- a/.gitattributes +++ b/.gitattributes @@ -40,6 +40,7 @@ huggingface_tokenizers_cache/models--unsloth--qwen2.5-1.5b-instruct-unsloth-bnb- results/phase1/checkpoint-100/tokenizer.json filter=lfs diff=lfs merge=lfs -text results/phase1/checkpoint-150/tokenizer.json filter=lfs diff=lfs merge=lfs -text results/phase1/checkpoint-200/tokenizer.json filter=lfs diff=lfs merge=lfs -text -results/phase1/checkpoint-222/tokenizer.json filter=lfs diff=lfs merge=lfs -text +results/phase1/checkpoint-225/tokenizer.json filter=lfs diff=lfs merge=lfs -text results/phase1/checkpoint-50/tokenizer.json filter=lfs diff=lfs merge=lfs -text +results/phase1/sft_warmup/checkpoint-48/tokenizer.json filter=lfs diff=lfs merge=lfs -text results/phase1/tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/results/phase1/README.md b/results/phase1/README.md index 64912959d6ae655109dc0817385276de06ed40eb..8a69192b51149a628589e7cc4f7b1d4af304e044 100644 --- a/results/phase1/README.md +++ b/results/phase1/README.md @@ -6,6 +6,7 @@ tags: - base_model:adapter:unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit - grpo - lora +- sft - transformers - trl - unsloth diff --git a/results/phase1/adapter_config.json b/results/phase1/adapter_config.json index 5e64881c1f2a3150228a608fb85ae9d9b6278fb9..d3cb3463c7f8c805e8cd8c25e64f3af87d6a00f3 100644 --- a/results/phase1/adapter_config.json +++ b/results/phase1/adapter_config.json @@ -33,13 +33,13 @@ "rank_pattern": {}, "revision": null, "target_modules": [ + "gate_proj", + "q_proj", "k_proj", - "up_proj", + "v_proj", "down_proj", "o_proj", - "v_proj", - "gate_proj", - "q_proj" + "up_proj" ], "target_parameters": null, "task_type": "CAUSAL_LM", diff --git a/results/phase1/adapter_model.safetensors b/results/phase1/adapter_model.safetensors index 5e624c334b6b7dbaef8cfa529b4c82b74e978fe7..09581caf47be4752636a0af973ec271be3c7854c 100644 --- a/results/phase1/adapter_model.safetensors +++ b/results/phase1/adapter_model.safetensors @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6d9ce380266a065d31b8d4962851f4d105daeda1ef56d4e9cd0835e5d5d0644a +oid sha256:fe1beb57ab89a2aeb9d572e64c9dedcaba34759e2e289c74a9a5464cb62a56c2 size 73911112 diff --git a/results/phase1/checkpoint-100/README.md b/results/phase1/checkpoint-100/README.md index 002e352e3f4655c9f00611b74eefa7b221963ec8..93a54877d7d12fb86dbb10111c6d07e97a31b3ea 100644 --- a/results/phase1/checkpoint-100/README.md +++ b/results/phase1/checkpoint-100/README.md @@ -6,6 +6,7 @@ tags: - base_model:adapter:unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit - grpo - lora +- sft - transformers - trl - unsloth diff --git a/results/phase1/checkpoint-100/adapter_config.json b/results/phase1/checkpoint-100/adapter_config.json index 5e64881c1f2a3150228a608fb85ae9d9b6278fb9..d3cb3463c7f8c805e8cd8c25e64f3af87d6a00f3 100644 --- a/results/phase1/checkpoint-100/adapter_config.json +++ b/results/phase1/checkpoint-100/adapter_config.json @@ -33,13 +33,13 @@ "rank_pattern": {}, "revision": null, "target_modules": [ + "gate_proj", + "q_proj", "k_proj", - "up_proj", + "v_proj", "down_proj", "o_proj", - "v_proj", - "gate_proj", - "q_proj" + "up_proj" ], "target_parameters": null, "task_type": "CAUSAL_LM", diff --git a/results/phase1/checkpoint-100/adapter_model.safetensors b/results/phase1/checkpoint-100/adapter_model.safetensors index 90c8bb55e3771db36f1a71bc7415777f30ffed49..253f185956cb9b91fe2975b6b97842c8ca508c8a 100644 --- a/results/phase1/checkpoint-100/adapter_model.safetensors +++ b/results/phase1/checkpoint-100/adapter_model.safetensors @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d430aea910e6d8b7347568b5b75ef11de9ff5b23b6d2010fd456033e1f218b07 +oid sha256:9ce4a1cadb39ce5fa4f01584e98f33cbc09dc8c4d7425778d7edc33616448be8 size 73911112 diff --git a/results/phase1/checkpoint-100/optimizer.pt b/results/phase1/checkpoint-100/optimizer.pt index 2c2d143a9948fdcefcacdc014234c92f0d096376..1a06597f59c8bc381d9875bfab39a388b56a1177 100644 --- a/results/phase1/checkpoint-100/optimizer.pt +++ b/results/phase1/checkpoint-100/optimizer.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:13de9f002823fbe63e71b9d1eb8ec6d2c4b8eca35a4ee367099322a459b3b1cb +oid sha256:7247a60399019dbce8eac389f415183266e38d6d0180c103cbec455696336c52 size 37969669 diff --git a/results/phase1/checkpoint-100/rng_state.pth b/results/phase1/checkpoint-100/rng_state.pth index 63e0b2eed0beee1d24ad3522e059120fa484e90f..e7f06d8cf3ab97a4251a7e2a040ab19d4f0f7511 100644 --- a/results/phase1/checkpoint-100/rng_state.pth +++ b/results/phase1/checkpoint-100/rng_state.pth @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c1449e60d52f06cceec8589785267b0457345be74a8ca37687099a6addff91e0 +oid sha256:f27587650ba6b8ea25bd2cc4a67db408173e91796efadb6b47615bd0c9322135 size 14645 diff --git a/results/phase1/checkpoint-100/scheduler.pt b/results/phase1/checkpoint-100/scheduler.pt index e7976eaf06add43be2f1a6fe7eb7b14a6720760a..d6e79c19955fc09a53badc8759c47ed887c7a377 100644 --- a/results/phase1/checkpoint-100/scheduler.pt +++ b/results/phase1/checkpoint-100/scheduler.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7254cd54e69ff9b68992c6e5838d1590c8e033e5a4605990d0b4034185923db6 +oid sha256:05a8fd04c04fd558f5a6d862a4a2344b4df262468cae8a6c245d1277ae908acb size 1465 diff --git a/results/phase1/checkpoint-100/trainer_state.json b/results/phase1/checkpoint-100/trainer_state.json index 95e17d555618ed2266692791d49264a512c75a8b..72eb510946d61c476eea894aeeb6f9b4b5b94f37 100644 --- a/results/phase1/checkpoint-100/trainer_state.json +++ b/results/phase1/checkpoint-100/trainer_state.json @@ -2,7 +2,7 @@ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, - "epoch": 0.45045045045045046, + "epoch": 0.4444444444444444, "eval_steps": 500, "global_step": 100, "is_hyper_param_search": false, @@ -15,25 +15,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 29.05, - "completions/clipped_ratio": 0.05, - "completions/max_length": 44.2, - "completions/max_terminated_length": 31.0, - "completions/mean_length": 29.05, - "completions/mean_terminated_length": 25.583333587646486, - "completions/min_length": 21.4, - "completions/min_terminated_length": 21.4, - "epoch": 0.02252252252252252, - "frac_reward_zero_std": 0.0, - "grad_norm": 1.2577366828918457, - "kl": 7.414049605358741e-06, + "completion_length": 25.075, + "completions/clipped_ratio": 0.0, + "completions/max_length": 30.8, + "completions/max_terminated_length": 30.8, + "completions/mean_length": 25.075, + "completions/mean_terminated_length": 25.075, + "completions/min_length": 19.4, + "completions/min_terminated_length": 19.4, + "epoch": 0.022222222222222223, + "frac_reward_zero_std": 0.6, + "grad_norm": 1.3092279434204102, + "kl": 0.03290070100920275, "learning_rate": 1.6666666666666667e-06, - "loss": 2.8312206268310547e-07, - "num_tokens": 21378.0, - "reward": 0.4944999933242798, - "reward_std": 0.15089658200740813, - "rewards/reward_fn/mean": 0.49449999928474425, - "rewards/reward_fn/std": 0.12999328523874282, + "loss": 0.0013160213828086853, + "num_tokens": 20275.0, + "reward": 0.48624999523162843, + "reward_std": 0.05833630859851837, + "rewards/reward_fn/mean": 0.4862500011920929, + "rewards/reward_fn/std": 0.04605271518230438, "step": 5 }, { @@ -42,25 +42,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 29.175, + "completion_length": 25.925, "completions/clipped_ratio": 0.0, - "completions/max_length": 41.8, - "completions/max_terminated_length": 41.8, - "completions/mean_length": 29.175, - "completions/mean_terminated_length": 29.175, - "completions/min_length": 19.2, - "completions/min_terminated_length": 19.2, - "epoch": 0.04504504504504504, + "completions/max_length": 33.0, + "completions/max_terminated_length": 33.0, + "completions/mean_length": 25.925, + "completions/mean_terminated_length": 25.925, + "completions/min_length": 19.4, + "completions/min_terminated_length": 19.4, + "epoch": 0.044444444444444446, "frac_reward_zero_std": 0.0, - "grad_norm": 1.148110032081604, - "kl": 2.58529256058182e-05, + "grad_norm": 1.4204450845718384, + "kl": 0.040759827199508436, "learning_rate": 3.7500000000000005e-06, - "loss": 9.343028068542481e-07, - "num_tokens": 42709.0, - "reward": 0.48589999675750734, - "reward_std": 0.14184561967849732, - "rewards/reward_fn/mean": 0.4859000027179718, - "rewards/reward_fn/std": 0.12539554834365846, + "loss": 0.0016303554177284241, + "num_tokens": 41560.0, + "reward": 0.5283999860286712, + "reward_std": 0.17366542518138886, + "rewards/reward_fn/mean": 0.5283999919891358, + "rewards/reward_fn/std": 0.16809422075748442, "step": 10 }, { @@ -69,25 +69,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 28.825, + "completion_length": 27.45, "completions/clipped_ratio": 0.0, - "completions/max_length": 44.2, - "completions/max_terminated_length": 44.2, - "completions/mean_length": 28.825, - "completions/mean_terminated_length": 28.825, - "completions/min_length": 21.2, - "completions/min_terminated_length": 21.2, - "epoch": 0.06756756756756757, + "completions/max_length": 33.6, + "completions/max_terminated_length": 33.6, + "completions/mean_length": 27.45, + "completions/mean_terminated_length": 27.45, + "completions/min_length": 24.0, + "completions/min_terminated_length": 24.0, + "epoch": 0.06666666666666667, "frac_reward_zero_std": 0.0, - "grad_norm": 1.1520825624465942, - "kl": 0.0001522944080875277, - "learning_rate": 4.952380952380953e-06, - "loss": 6.105005741119385e-06, - "num_tokens": 63008.0, - "reward": 0.4894999861717224, - "reward_std": 0.15768481492996217, - "rewards/reward_fn/mean": 0.48950000405311583, - "rewards/reward_fn/std": 0.14185291826725005, + "grad_norm": 0.9515441656112671, + "kl": 0.049849181214813146, + "learning_rate": 4.953051643192488e-06, + "loss": 0.0019938603043556215, + "num_tokens": 62246.0, + "reward": 0.4975499987602234, + "reward_std": 0.1564827263355255, + "rewards/reward_fn/mean": 0.49755001068115234, + "rewards/reward_fn/std": 0.14618260115385057, "step": 15 }, { @@ -96,25 +96,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 28.25, + "completion_length": 26.525, "completions/clipped_ratio": 0.0, - "completions/max_length": 43.2, - "completions/max_terminated_length": 43.2, - "completions/mean_length": 28.25, - "completions/mean_terminated_length": 28.25, - "completions/min_length": 21.6, - "completions/min_terminated_length": 21.6, - "epoch": 0.09009009009009009, + "completions/max_length": 33.0, + "completions/max_terminated_length": 33.0, + "completions/mean_length": 26.525, + "completions/mean_terminated_length": 26.525, + "completions/min_length": 22.2, + "completions/min_terminated_length": 22.2, + "epoch": 0.08888888888888889, "frac_reward_zero_std": 0.0, - "grad_norm": 1.0063226222991943, - "kl": 0.0008983879492006963, - "learning_rate": 4.833333333333333e-06, - "loss": 3.593862056732178e-05, - "num_tokens": 82836.0, - "reward": 0.4881500005722046, - "reward_std": 0.14941166192293168, - "rewards/reward_fn/mean": 0.48814999461174013, - "rewards/reward_fn/std": 0.12791907638311387, + "grad_norm": 0.6862999200820923, + "kl": 0.04350169296376407, + "learning_rate": 4.835680751173709e-06, + "loss": 0.0017400771379470826, + "num_tokens": 81479.0, + "reward": 0.5159000098705292, + "reward_std": 0.17465537190437316, + "rewards/reward_fn/mean": 0.5158999919891357, + "rewards/reward_fn/std": 0.16455023884773254, "step": 20 }, { @@ -123,25 +123,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 27.25, + "completion_length": 28.0, "completions/clipped_ratio": 0.0, - "completions/max_length": 45.6, - "completions/max_terminated_length": 45.6, - "completions/mean_length": 27.25, - "completions/mean_terminated_length": 27.25, - "completions/min_length": 18.8, - "completions/min_terminated_length": 18.8, - "epoch": 0.11261261261261261, + "completions/max_length": 40.6, + "completions/max_terminated_length": 40.6, + "completions/mean_length": 28.0, + "completions/mean_terminated_length": 28.0, + "completions/min_length": 22.6, + "completions/min_terminated_length": 22.6, + "epoch": 0.1111111111111111, "frac_reward_zero_std": 0.0, - "grad_norm": 1.3132057189941406, - "kl": 0.003781939600594342, - "learning_rate": 4.714285714285715e-06, - "loss": 0.00015127062797546387, - "num_tokens": 103596.0, - "reward": 0.4680500030517578, - "reward_std": 0.17189764976501465, - "rewards/reward_fn/mean": 0.46804999113082885, - "rewards/reward_fn/std": 0.17421672195196153, + "grad_norm": 1.7516958713531494, + "kl": 0.05824573401478119, + "learning_rate": 4.71830985915493e-06, + "loss": 0.002329717576503754, + "num_tokens": 102833.0, + "reward": 0.5547999978065491, + "reward_std": 0.18328206837177277, + "rewards/reward_fn/mean": 0.5547999858856201, + "rewards/reward_fn/std": 0.1974634051322937, "step": 25 }, { @@ -150,25 +150,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 25.225, + "completion_length": 28.675, "completions/clipped_ratio": 0.0, - "completions/max_length": 34.6, - "completions/max_terminated_length": 34.6, - "completions/mean_length": 25.225, - "completions/mean_terminated_length": 25.225, - "completions/min_length": 19.2, - "completions/min_terminated_length": 19.2, - "epoch": 0.13513513513513514, + "completions/max_length": 37.4, + "completions/max_terminated_length": 37.4, + "completions/mean_length": 28.675, + "completions/mean_terminated_length": 28.675, + "completions/min_length": 22.2, + "completions/min_terminated_length": 22.2, + "epoch": 0.13333333333333333, "frac_reward_zero_std": 0.0, - "grad_norm": 1.5750545263290405, - "kl": 0.006673775642411783, - "learning_rate": 4.595238095238095e-06, - "loss": 0.0002669498324394226, - "num_tokens": 123651.0, - "reward": 0.48049998879432676, - "reward_std": 0.1656044065952301, - "rewards/reward_fn/mean": 0.48049998879432676, - "rewards/reward_fn/std": 0.13908967524766921, + "grad_norm": 1.5611271858215332, + "kl": 0.09752920938190072, + "learning_rate": 4.60093896713615e-06, + "loss": 0.0039011374115943908, + "num_tokens": 123876.0, + "reward": 0.5165499806404114, + "reward_std": 0.16765501499176025, + "rewards/reward_fn/mean": 0.5165499806404114, + "rewards/reward_fn/std": 0.16748485416173936, "step": 30 }, { @@ -177,25 +177,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 28.775, - "completions/clipped_ratio": 0.025, - "completions/max_length": 54.8, - "completions/max_terminated_length": 41.8, - "completions/mean_length": 28.775, - "completions/mean_terminated_length": 26.967857360839844, - "completions/min_length": 15.4, - "completions/min_terminated_length": 15.4, - "epoch": 0.15765765765765766, + "completion_length": 27.475, + "completions/clipped_ratio": 0.0, + "completions/max_length": 36.8, + "completions/max_terminated_length": 36.8, + "completions/mean_length": 27.475, + "completions/mean_terminated_length": 27.475, + "completions/min_length": 20.6, + "completions/min_terminated_length": 20.6, + "epoch": 0.15555555555555556, "frac_reward_zero_std": 0.0, - "grad_norm": 1.1105402708053589, - "kl": 0.005106111426721327, - "learning_rate": 4.476190476190477e-06, - "loss": 0.0002042025327682495, - "num_tokens": 144316.0, - "reward": 0.4778999924659729, - "reward_std": 0.17012988924980163, - "rewards/reward_fn/mean": 0.47790001034736634, - "rewards/reward_fn/std": 0.14147266000509262, + "grad_norm": 0.9550014138221741, + "kl": 0.08032013729680329, + "learning_rate": 4.483568075117371e-06, + "loss": 0.003212757408618927, + "num_tokens": 145019.0, + "reward": 0.5004499793052674, + "reward_std": 0.18391846716403962, + "rewards/reward_fn/mean": 0.5004500031471253, + "rewards/reward_fn/std": 0.16948954164981841, "step": 35 }, { @@ -204,25 +204,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 24.4, - "completions/clipped_ratio": 0.0, - "completions/max_length": 33.4, - "completions/max_terminated_length": 33.4, - "completions/mean_length": 24.4, - "completions/mean_terminated_length": 24.4, - "completions/min_length": 18.8, - "completions/min_terminated_length": 18.8, - "epoch": 0.18018018018018017, + "completion_length": 32.125, + "completions/clipped_ratio": 0.025, + "completions/max_length": 54.6, + "completions/max_terminated_length": 46.0, + "completions/mean_length": 32.125, + "completions/mean_terminated_length": 30.521428680419923, + "completions/min_length": 23.2, + "completions/min_terminated_length": 23.2, + "epoch": 0.17777777777777778, "frac_reward_zero_std": 0.0, - "grad_norm": 1.0676746368408203, - "kl": 0.006288998411764624, - "learning_rate": 4.357142857142857e-06, - "loss": 0.00025154203176498414, - "num_tokens": 164334.0, - "reward": 0.5014999985694886, - "reward_std": 0.15358359068632127, - "rewards/reward_fn/mean": 0.5014999985694886, - "rewards/reward_fn/std": 0.13824734836816788, + "grad_norm": 1.1203869581222534, + "kl": 0.08563535290304572, + "learning_rate": 4.3661971830985915e-06, + "loss": 0.003425435721874237, + "num_tokens": 164800.0, + "reward": 0.5472500026226044, + "reward_std": 0.19579786211252212, + "rewards/reward_fn/mean": 0.5472500085830688, + "rewards/reward_fn/std": 0.19857290089130403, "step": 40 }, { @@ -231,25 +231,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 27.35, + "completion_length": 27.875, "completions/clipped_ratio": 0.0, - "completions/max_length": 43.6, - "completions/max_terminated_length": 43.6, - "completions/mean_length": 27.35, - "completions/mean_terminated_length": 27.35, - "completions/min_length": 19.6, - "completions/min_terminated_length": 19.6, - "epoch": 0.20270270270270271, + "completions/max_length": 35.0, + "completions/max_terminated_length": 35.0, + "completions/mean_length": 27.875, + "completions/mean_terminated_length": 27.875, + "completions/min_length": 21.4, + "completions/min_terminated_length": 21.4, + "epoch": 0.2, "frac_reward_zero_std": 0.0, - "grad_norm": 1.0749404430389404, - "kl": 0.010629063473606948, - "learning_rate": 4.238095238095239e-06, - "loss": 0.0004251018166542053, - "num_tokens": 185140.0, - "reward": 0.524949985742569, - "reward_std": 0.14630039632320405, - "rewards/reward_fn/mean": 0.5249499917030335, - "rewards/reward_fn/std": 0.1460244983434677, + "grad_norm": 0.95042884349823, + "kl": 0.07295196709455923, + "learning_rate": 4.248826291079813e-06, + "loss": 0.0029180020093917845, + "num_tokens": 185761.0, + "reward": 0.5225999832153321, + "reward_std": 0.16489729881286622, + "rewards/reward_fn/mean": 0.5226000070571899, + "rewards/reward_fn/std": 0.16541497856378556, "step": 45 }, { @@ -258,25 +258,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 25.425, + "completion_length": 27.875, "completions/clipped_ratio": 0.0, "completions/max_length": 35.2, "completions/max_terminated_length": 35.2, - "completions/mean_length": 25.425, - "completions/mean_terminated_length": 25.425, - "completions/min_length": 17.6, - "completions/min_terminated_length": 17.6, - "epoch": 0.22522522522522523, + "completions/mean_length": 27.875, + "completions/mean_terminated_length": 27.875, + "completions/min_length": 21.4, + "completions/min_terminated_length": 21.4, + "epoch": 0.2222222222222222, "frac_reward_zero_std": 0.0, - "grad_norm": 1.4353065490722656, - "kl": 0.008290678875346203, - "learning_rate": 4.119047619047619e-06, - "loss": 0.0003315746784210205, - "num_tokens": 206569.0, - "reward": 0.4908999800682068, - "reward_std": 0.17055415213108063, - "rewards/reward_fn/mean": 0.49089999198913575, - "rewards/reward_fn/std": 0.154354290664196, + "grad_norm": 1.1037548780441284, + "kl": 0.08246680488809943, + "learning_rate": 4.131455399061034e-06, + "loss": 0.0032985761761665346, + "num_tokens": 206674.0, + "reward": 0.5245000004768372, + "reward_std": 0.1653215616941452, + "rewards/reward_fn/mean": 0.5245000004768372, + "rewards/reward_fn/std": 0.16877340227365495, "step": 50 }, { @@ -285,25 +285,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 27.7, - "completions/clipped_ratio": 0.025, - "completions/max_length": 50.6, - "completions/max_terminated_length": 37.2, - "completions/mean_length": 27.7, - "completions/mean_terminated_length": 25.91071434020996, - "completions/min_length": 18.2, - "completions/min_terminated_length": 18.2, - "epoch": 0.24774774774774774, + "completion_length": 29.225, + "completions/clipped_ratio": 0.0, + "completions/max_length": 47.4, + "completions/max_terminated_length": 47.4, + "completions/mean_length": 29.225, + "completions/mean_terminated_length": 29.225, + "completions/min_length": 20.2, + "completions/min_terminated_length": 20.2, + "epoch": 0.24444444444444444, "frac_reward_zero_std": 0.0, - "grad_norm": 0.8640509843826294, - "kl": 0.012456063569698018, - "learning_rate": 4.000000000000001e-06, - "loss": 0.0004982292652130127, - "num_tokens": 226097.0, - "reward": 0.4839499771595001, - "reward_std": 0.1590283066034317, - "rewards/reward_fn/mean": 0.48394999504089353, - "rewards/reward_fn/std": 0.1380786642432213, + "grad_norm": 0.8115681409835815, + "kl": 0.08297618771903217, + "learning_rate": 4.014084507042254e-06, + "loss": 0.0033189669251441956, + "num_tokens": 227351.0, + "reward": 0.5301500022411346, + "reward_std": 0.20612163245677947, + "rewards/reward_fn/mean": 0.5301500082015991, + "rewards/reward_fn/std": 0.19000594317913055, "step": 55 }, { @@ -312,25 +312,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 26.875, - "completions/clipped_ratio": 0.0, - "completions/max_length": 40.0, - "completions/max_terminated_length": 40.0, - "completions/mean_length": 26.875, - "completions/mean_terminated_length": 26.875, - "completions/min_length": 20.6, - "completions/min_terminated_length": 20.6, - "epoch": 0.2702702702702703, + "completion_length": 27.475, + "completions/clipped_ratio": 0.025, + "completions/max_length": 45.0, + "completions/max_terminated_length": 31.8, + "completions/mean_length": 27.475, + "completions/mean_terminated_length": 25.646428680419923, + "completions/min_length": 18.8, + "completions/min_terminated_length": 18.8, + "epoch": 0.26666666666666666, "frac_reward_zero_std": 0.0, - "grad_norm": 1.0529346466064453, - "kl": 0.007519814093393506, - "learning_rate": 3.880952380952381e-06, - "loss": 0.00030075013637542725, - "num_tokens": 246836.0, - "reward": 0.4914499819278717, - "reward_std": 0.1727461814880371, - "rewards/reward_fn/mean": 0.49144998788833616, - "rewards/reward_fn/std": 0.1568704679608345, + "grad_norm": 1.1722970008850098, + "kl": 0.06283304298995063, + "learning_rate": 3.896713615023475e-06, + "loss": 0.002513286471366882, + "num_tokens": 246464.0, + "reward": 0.513349997997284, + "reward_std": 0.18999958634376526, + "rewards/reward_fn/mean": 0.5133499920368194, + "rewards/reward_fn/std": 0.1748345360159874, "step": 60 }, { @@ -339,25 +339,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 27.825, - "completions/clipped_ratio": 0.025, - "completions/max_length": 51.2, - "completions/max_terminated_length": 38.0, - "completions/mean_length": 27.825, - "completions/mean_terminated_length": 26.03214302062988, - "completions/min_length": 17.6, - "completions/min_terminated_length": 17.6, - "epoch": 0.2927927927927928, + "completion_length": 26.075, + "completions/clipped_ratio": 0.0, + "completions/max_length": 34.2, + "completions/max_terminated_length": 34.2, + "completions/mean_length": 26.075, + "completions/mean_terminated_length": 26.075, + "completions/min_length": 19.4, + "completions/min_terminated_length": 19.4, + "epoch": 0.28888888888888886, "frac_reward_zero_std": 0.0, - "grad_norm": 1.149415373802185, - "kl": 0.00938094645498495, - "learning_rate": 3.761904761904762e-06, - "loss": 0.0003751114010810852, - "num_tokens": 266985.0, - "reward": 0.5003499746322632, - "reward_std": 0.16892780363559723, - "rewards/reward_fn/mean": 0.5003500044345855, - "rewards/reward_fn/std": 0.14663992822170258, + "grad_norm": 1.2825119495391846, + "kl": 0.07132846353342756, + "learning_rate": 3.779342723004695e-06, + "loss": 0.0028530970215797425, + "num_tokens": 267279.0, + "reward": 0.5383000016212464, + "reward_std": 0.20746512711048126, + "rewards/reward_fn/mean": 0.5383000195026397, + "rewards/reward_fn/std": 0.1835445523262024, "step": 65 }, { @@ -366,25 +366,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 27.0, + "completion_length": 27.925, "completions/clipped_ratio": 0.0, - "completions/max_length": 40.6, - "completions/max_terminated_length": 40.6, - "completions/mean_length": 27.0, - "completions/mean_terminated_length": 27.0, - "completions/min_length": 17.8, - "completions/min_terminated_length": 17.8, - "epoch": 0.3153153153153153, + "completions/max_length": 44.0, + "completions/max_terminated_length": 44.0, + "completions/mean_length": 27.925, + "completions/mean_terminated_length": 27.925, + "completions/min_length": 22.0, + "completions/min_terminated_length": 22.0, + "epoch": 0.3111111111111111, "frac_reward_zero_std": 0.0, - "grad_norm": 0.9893133640289307, - "kl": 0.010053556466664304, - "learning_rate": 3.642857142857143e-06, - "loss": 0.00040204524993896483, - "num_tokens": 288099.0, - "reward": 0.4705499827861786, - "reward_std": 0.16256383657455445, - "rewards/reward_fn/mean": 0.470550000667572, - "rewards/reward_fn/std": 0.13094386458396912, + "grad_norm": 0.5832120776176453, + "kl": 0.05197672065114602, + "learning_rate": 3.6619718309859158e-06, + "loss": 0.0020790368318557738, + "num_tokens": 287984.0, + "reward": 0.5424999892711639, + "reward_std": 0.18002938330173493, + "rewards/reward_fn/mean": 0.5424999952316284, + "rewards/reward_fn/std": 0.18769851326942444, "step": 70 }, { @@ -393,25 +393,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 25.75, - "completions/clipped_ratio": 0.0, - "completions/max_length": 34.8, - "completions/max_terminated_length": 34.8, - "completions/mean_length": 25.75, - "completions/mean_terminated_length": 25.75, - "completions/min_length": 19.8, - "completions/min_terminated_length": 19.8, - "epoch": 0.33783783783783783, + "completion_length": 27.6, + "completions/clipped_ratio": 0.025, + "completions/max_length": 44.6, + "completions/max_terminated_length": 30.6, + "completions/mean_length": 27.6, + "completions/mean_terminated_length": 25.807143020629884, + "completions/min_length": 22.2, + "completions/min_terminated_length": 22.2, + "epoch": 0.3333333333333333, "frac_reward_zero_std": 0.0, - "grad_norm": 1.063340187072754, - "kl": 0.008550376921630232, - "learning_rate": 3.523809523809524e-06, - "loss": 0.00034204572439193725, - "num_tokens": 308531.0, - "reward": 0.5000500082969666, - "reward_std": 0.16242243051528932, - "rewards/reward_fn/mean": 0.5000500082969666, - "rewards/reward_fn/std": 0.14417539685964584, + "grad_norm": 0.6677760481834412, + "kl": 0.03671608620206825, + "learning_rate": 3.5446009389671364e-06, + "loss": 0.0014685407280921937, + "num_tokens": 307800.0, + "reward": 0.5715999841690064, + "reward_std": 0.18639334440231323, + "rewards/reward_fn/mean": 0.5715999960899353, + "rewards/reward_fn/std": 0.19429495334625244, "step": 75 }, { @@ -420,25 +420,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 27.025, + "completion_length": 25.6, "completions/clipped_ratio": 0.0, - "completions/max_length": 40.0, - "completions/max_terminated_length": 40.0, - "completions/mean_length": 27.025, - "completions/mean_terminated_length": 27.025, - "completions/min_length": 19.4, - "completions/min_terminated_length": 19.4, - "epoch": 0.36036036036036034, + "completions/max_length": 30.2, + "completions/max_terminated_length": 30.2, + "completions/mean_length": 25.6, + "completions/mean_terminated_length": 25.6, + "completions/min_length": 21.4, + "completions/min_terminated_length": 21.4, + "epoch": 0.35555555555555557, "frac_reward_zero_std": 0.0, - "grad_norm": 1.2858343124389648, - "kl": 0.019255010827328077, - "learning_rate": 3.404761904761905e-06, - "loss": 0.0007702425122261047, - "num_tokens": 328874.0, - "reward": 0.5261500000953674, - "reward_std": 0.11886464804410934, - "rewards/reward_fn/mean": 0.5261499881744385, - "rewards/reward_fn/std": 0.12691220343112947, + "grad_norm": 0.7261527180671692, + "kl": 0.05616153636947274, + "learning_rate": 3.427230046948357e-06, + "loss": 0.0022463813424110413, + "num_tokens": 327794.0, + "reward": 0.5464499950408935, + "reward_std": 0.18264567852020264, + "rewards/reward_fn/mean": 0.5464499950408935, + "rewards/reward_fn/std": 0.17480863779783248, "step": 80 }, { @@ -447,25 +447,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 25.475, + "completion_length": 24.075, "completions/clipped_ratio": 0.0, - "completions/max_length": 33.0, - "completions/max_terminated_length": 33.0, - "completions/mean_length": 25.475, - "completions/mean_terminated_length": 25.475, - "completions/min_length": 19.0, - "completions/min_terminated_length": 19.0, - "epoch": 0.38288288288288286, + "completions/max_length": 29.6, + "completions/max_terminated_length": 29.6, + "completions/mean_length": 24.075, + "completions/mean_terminated_length": 24.075, + "completions/min_length": 16.6, + "completions/min_terminated_length": 16.6, + "epoch": 0.37777777777777777, "frac_reward_zero_std": 0.0, - "grad_norm": 1.118030309677124, - "kl": 0.01090274843845691, - "learning_rate": 3.285714285714286e-06, - "loss": 0.00043606162071228025, - "num_tokens": 348373.0, - "reward": 0.49950000643730164, - "reward_std": 0.15655344128608703, - "rewards/reward_fn/mean": 0.49950000643730164, - "rewards/reward_fn/std": 0.14320258051156998, + "grad_norm": 1.1086981296539307, + "kl": 0.07743949705036357, + "learning_rate": 3.3098591549295777e-06, + "loss": 0.0030974715948104857, + "num_tokens": 348233.0, + "reward": 0.5536999821662902, + "reward_std": 0.18073648810386658, + "rewards/reward_fn/mean": 0.5536999821662902, + "rewards/reward_fn/std": 0.1820658951997757, "step": 85 }, { @@ -474,25 +474,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 27.9, - "completions/clipped_ratio": 0.05, - "completions/max_length": 55.4, - "completions/max_terminated_length": 31.6, - "completions/mean_length": 27.9, - "completions/mean_terminated_length": 24.28571472167969, - "completions/min_length": 18.0, - "completions/min_terminated_length": 18.0, - "epoch": 0.40540540540540543, + "completion_length": 25.375, + "completions/clipped_ratio": 0.0, + "completions/max_length": 34.4, + "completions/max_terminated_length": 34.4, + "completions/mean_length": 25.375, + "completions/mean_terminated_length": 25.375, + "completions/min_length": 18.6, + "completions/min_terminated_length": 18.6, + "epoch": 0.4, "frac_reward_zero_std": 0.0, - "grad_norm": 0.6538166403770447, - "kl": 0.008556566922561615, - "learning_rate": 3.1666666666666667e-06, - "loss": 0.0003422081470489502, - "num_tokens": 368815.0, - "reward": 0.5215500056743622, - "reward_std": 0.16581654250621797, - "rewards/reward_fn/mean": 0.5215500056743622, - "rewards/reward_fn/std": 0.15472394227981567, + "grad_norm": 1.1903071403503418, + "kl": 0.058934826811309904, + "learning_rate": 3.1924882629107983e-06, + "loss": 0.002357317507266998, + "num_tokens": 367732.0, + "reward": 0.5361499905586242, + "reward_std": 0.179675829410553, + "rewards/reward_fn/mean": 0.5361500144004822, + "rewards/reward_fn/std": 0.18574500381946563, "step": 90 }, { @@ -501,25 +501,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 25.75, + "completion_length": 26.825, "completions/clipped_ratio": 0.0, - "completions/max_length": 32.4, - "completions/max_terminated_length": 32.4, - "completions/mean_length": 25.75, - "completions/mean_terminated_length": 25.75, + "completions/max_length": 37.2, + "completions/max_terminated_length": 37.2, + "completions/mean_length": 26.825, + "completions/mean_terminated_length": 26.825, "completions/min_length": 20.8, "completions/min_terminated_length": 20.8, - "epoch": 0.42792792792792794, + "epoch": 0.4222222222222222, "frac_reward_zero_std": 0.0, - "grad_norm": 1.180251955986023, - "kl": 0.014757526386529208, - "learning_rate": 3.047619047619048e-06, - "loss": 0.0005902737379074096, - "num_tokens": 388761.0, - "reward": 0.514000016450882, - "reward_std": 0.1465125188231468, - "rewards/reward_fn/mean": 0.5139999985694885, - "rewards/reward_fn/std": 0.14701166301965712, + "grad_norm": 1.0952492952346802, + "kl": 0.040232469444163144, + "learning_rate": 3.075117370892019e-06, + "loss": 0.0016092658042907715, + "num_tokens": 387075.0, + "reward": 0.5278500020503998, + "reward_std": 0.1932522773742676, + "rewards/reward_fn/mean": 0.5278499901294709, + "rewards/reward_fn/std": 0.1796583503484726, "step": 95 }, { @@ -528,31 +528,31 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 26.35, - "completions/clipped_ratio": 0.025, - "completions/max_length": 46.6, - "completions/max_terminated_length": 32.4, - "completions/mean_length": 26.35, - "completions/mean_terminated_length": 24.517857360839844, - "completions/min_length": 17.8, - "completions/min_terminated_length": 17.8, - "epoch": 0.45045045045045046, + "completion_length": 26.125, + "completions/clipped_ratio": 0.0, + "completions/max_length": 35.6, + "completions/max_terminated_length": 35.6, + "completions/mean_length": 26.125, + "completions/mean_terminated_length": 26.125, + "completions/min_length": 22.6, + "completions/min_terminated_length": 22.6, + "epoch": 0.4444444444444444, "frac_reward_zero_std": 0.0, - "grad_norm": 0.8863919973373413, - "kl": 0.017311154678463937, - "learning_rate": 2.928571428571429e-06, - "loss": 0.0006924748420715332, - "num_tokens": 408309.0, - "reward": 0.5527999818325042, - "reward_std": 0.1593818634748459, - "rewards/reward_fn/mean": 0.5527999699115753, - "rewards/reward_fn/std": 0.16823574155569077, + "grad_norm": 0.8072858452796936, + "kl": 0.06188266044482589, + "learning_rate": 2.9577464788732396e-06, + "loss": 0.0024752289056777952, + "num_tokens": 406884.0, + "reward": 0.5849499881267548, + "reward_std": 0.20499025285243988, + "rewards/reward_fn/mean": 0.5849500000476837, + "rewards/reward_fn/std": 0.2056175708770752, "step": 100 } ], "logging_steps": 5, - "max_steps": 222, - "num_input_tokens_seen": 408309, + "max_steps": 225, + "num_input_tokens_seen": 406884, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { diff --git a/results/phase1/checkpoint-150/README.md b/results/phase1/checkpoint-150/README.md index 002e352e3f4655c9f00611b74eefa7b221963ec8..93a54877d7d12fb86dbb10111c6d07e97a31b3ea 100644 --- a/results/phase1/checkpoint-150/README.md +++ b/results/phase1/checkpoint-150/README.md @@ -6,6 +6,7 @@ tags: - base_model:adapter:unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit - grpo - lora +- sft - transformers - trl - unsloth diff --git a/results/phase1/checkpoint-150/adapter_config.json b/results/phase1/checkpoint-150/adapter_config.json index 5e64881c1f2a3150228a608fb85ae9d9b6278fb9..d3cb3463c7f8c805e8cd8c25e64f3af87d6a00f3 100644 --- a/results/phase1/checkpoint-150/adapter_config.json +++ b/results/phase1/checkpoint-150/adapter_config.json @@ -33,13 +33,13 @@ "rank_pattern": {}, "revision": null, "target_modules": [ + "gate_proj", + "q_proj", "k_proj", - "up_proj", + "v_proj", "down_proj", "o_proj", - "v_proj", - "gate_proj", - "q_proj" + "up_proj" ], "target_parameters": null, "task_type": "CAUSAL_LM", diff --git a/results/phase1/checkpoint-150/adapter_model.safetensors b/results/phase1/checkpoint-150/adapter_model.safetensors index e22a8281f61e53fda46d6dd06144dff42abd2e9b..af58c13a887a396c2b37ccf9a7aa8099c7cbb80e 100644 --- a/results/phase1/checkpoint-150/adapter_model.safetensors +++ b/results/phase1/checkpoint-150/adapter_model.safetensors @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4f2ea6f37aa971ffcfb6e901dff891103ce7361942994c3c9c4764d57335315a +oid sha256:32acfb9063f0acb0116e611afccc783f35f4c73cff621c31498366c2b5a56064 size 73911112 diff --git a/results/phase1/checkpoint-150/optimizer.pt b/results/phase1/checkpoint-150/optimizer.pt index ba1a5d1ed652c98acb966f616bc7cd67338b1def..420cdbd89d1f3040ac9747d885c8b12292f778ff 100644 --- a/results/phase1/checkpoint-150/optimizer.pt +++ b/results/phase1/checkpoint-150/optimizer.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ac5a839e2d17ed3303e7dcd56b82670c235a4599960cf3f5bed14b6d0184b795 +oid sha256:4f3f1173d331f3f50e08ce052b94de7042ea5521c647c894e9b2fb1fc318e914 size 37969669 diff --git a/results/phase1/checkpoint-150/rng_state.pth b/results/phase1/checkpoint-150/rng_state.pth index 67579e3a222fd329f15f18191f0f913654b83ff9..952f8aac05bf898f7231a1f0d03dd87e1090b8e8 100644 --- a/results/phase1/checkpoint-150/rng_state.pth +++ b/results/phase1/checkpoint-150/rng_state.pth @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:df9f68fada42e0a51f4516e482fb448af8d3c5d6419e5f87e2cb03ca9217eaef +oid sha256:b6ca1b31c2efde8457f21b4590f2ea6c56859d8863161b4e4a23778d49b3bda3 size 14645 diff --git a/results/phase1/checkpoint-150/scheduler.pt b/results/phase1/checkpoint-150/scheduler.pt index 291d63c29103b6743ac4f2028a586f93cc706670..92847edd7069463b9915db1a94a46d5e3833914c 100644 --- a/results/phase1/checkpoint-150/scheduler.pt +++ b/results/phase1/checkpoint-150/scheduler.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f50a1d8632a9c468db3f4baf075e2db44d9458b659d8b9169b83f39c755aa704 +oid sha256:c67b5ab96e955e978a225afc8c7ba252edc2deb95112be875bab2391d90e77ab size 1465 diff --git a/results/phase1/checkpoint-150/trainer_state.json b/results/phase1/checkpoint-150/trainer_state.json index c64c8559e003be5296ccaa0b74221b3b10e92c87..0e912b423b117211b511bc1089e7baf7b2387b85 100644 --- a/results/phase1/checkpoint-150/trainer_state.json +++ b/results/phase1/checkpoint-150/trainer_state.json @@ -2,7 +2,7 @@ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, - "epoch": 0.6756756756756757, + "epoch": 0.6666666666666666, "eval_steps": 500, "global_step": 150, "is_hyper_param_search": false, @@ -15,25 +15,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 29.05, - "completions/clipped_ratio": 0.05, - "completions/max_length": 44.2, - "completions/max_terminated_length": 31.0, - "completions/mean_length": 29.05, - "completions/mean_terminated_length": 25.583333587646486, - "completions/min_length": 21.4, - "completions/min_terminated_length": 21.4, - "epoch": 0.02252252252252252, - "frac_reward_zero_std": 0.0, - "grad_norm": 1.2577366828918457, - "kl": 7.414049605358741e-06, + "completion_length": 25.075, + "completions/clipped_ratio": 0.0, + "completions/max_length": 30.8, + "completions/max_terminated_length": 30.8, + "completions/mean_length": 25.075, + "completions/mean_terminated_length": 25.075, + "completions/min_length": 19.4, + "completions/min_terminated_length": 19.4, + "epoch": 0.022222222222222223, + "frac_reward_zero_std": 0.6, + "grad_norm": 1.3092279434204102, + "kl": 0.03290070100920275, "learning_rate": 1.6666666666666667e-06, - "loss": 2.8312206268310547e-07, - "num_tokens": 21378.0, - "reward": 0.4944999933242798, - "reward_std": 0.15089658200740813, - "rewards/reward_fn/mean": 0.49449999928474425, - "rewards/reward_fn/std": 0.12999328523874282, + "loss": 0.0013160213828086853, + "num_tokens": 20275.0, + "reward": 0.48624999523162843, + "reward_std": 0.05833630859851837, + "rewards/reward_fn/mean": 0.4862500011920929, + "rewards/reward_fn/std": 0.04605271518230438, "step": 5 }, { @@ -42,25 +42,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 29.175, + "completion_length": 25.925, "completions/clipped_ratio": 0.0, - "completions/max_length": 41.8, - "completions/max_terminated_length": 41.8, - "completions/mean_length": 29.175, - "completions/mean_terminated_length": 29.175, - "completions/min_length": 19.2, - "completions/min_terminated_length": 19.2, - "epoch": 0.04504504504504504, + "completions/max_length": 33.0, + "completions/max_terminated_length": 33.0, + "completions/mean_length": 25.925, + "completions/mean_terminated_length": 25.925, + "completions/min_length": 19.4, + "completions/min_terminated_length": 19.4, + "epoch": 0.044444444444444446, "frac_reward_zero_std": 0.0, - "grad_norm": 1.148110032081604, - "kl": 2.58529256058182e-05, + "grad_norm": 1.4204450845718384, + "kl": 0.040759827199508436, "learning_rate": 3.7500000000000005e-06, - "loss": 9.343028068542481e-07, - "num_tokens": 42709.0, - "reward": 0.48589999675750734, - "reward_std": 0.14184561967849732, - "rewards/reward_fn/mean": 0.4859000027179718, - "rewards/reward_fn/std": 0.12539554834365846, + "loss": 0.0016303554177284241, + "num_tokens": 41560.0, + "reward": 0.5283999860286712, + "reward_std": 0.17366542518138886, + "rewards/reward_fn/mean": 0.5283999919891358, + "rewards/reward_fn/std": 0.16809422075748442, "step": 10 }, { @@ -69,25 +69,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 28.825, + "completion_length": 27.45, "completions/clipped_ratio": 0.0, - "completions/max_length": 44.2, - "completions/max_terminated_length": 44.2, - "completions/mean_length": 28.825, - "completions/mean_terminated_length": 28.825, - "completions/min_length": 21.2, - "completions/min_terminated_length": 21.2, - "epoch": 0.06756756756756757, + "completions/max_length": 33.6, + "completions/max_terminated_length": 33.6, + "completions/mean_length": 27.45, + "completions/mean_terminated_length": 27.45, + "completions/min_length": 24.0, + "completions/min_terminated_length": 24.0, + "epoch": 0.06666666666666667, "frac_reward_zero_std": 0.0, - "grad_norm": 1.1520825624465942, - "kl": 0.0001522944080875277, - "learning_rate": 4.952380952380953e-06, - "loss": 6.105005741119385e-06, - "num_tokens": 63008.0, - "reward": 0.4894999861717224, - "reward_std": 0.15768481492996217, - "rewards/reward_fn/mean": 0.48950000405311583, - "rewards/reward_fn/std": 0.14185291826725005, + "grad_norm": 0.9515441656112671, + "kl": 0.049849181214813146, + "learning_rate": 4.953051643192488e-06, + "loss": 0.0019938603043556215, + "num_tokens": 62246.0, + "reward": 0.4975499987602234, + "reward_std": 0.1564827263355255, + "rewards/reward_fn/mean": 0.49755001068115234, + "rewards/reward_fn/std": 0.14618260115385057, "step": 15 }, { @@ -96,25 +96,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 28.25, + "completion_length": 26.525, "completions/clipped_ratio": 0.0, - "completions/max_length": 43.2, - "completions/max_terminated_length": 43.2, - "completions/mean_length": 28.25, - "completions/mean_terminated_length": 28.25, - "completions/min_length": 21.6, - "completions/min_terminated_length": 21.6, - "epoch": 0.09009009009009009, + "completions/max_length": 33.0, + "completions/max_terminated_length": 33.0, + "completions/mean_length": 26.525, + "completions/mean_terminated_length": 26.525, + "completions/min_length": 22.2, + "completions/min_terminated_length": 22.2, + "epoch": 0.08888888888888889, "frac_reward_zero_std": 0.0, - "grad_norm": 1.0063226222991943, - "kl": 0.0008983879492006963, - "learning_rate": 4.833333333333333e-06, - "loss": 3.593862056732178e-05, - "num_tokens": 82836.0, - "reward": 0.4881500005722046, - "reward_std": 0.14941166192293168, - "rewards/reward_fn/mean": 0.48814999461174013, - "rewards/reward_fn/std": 0.12791907638311387, + "grad_norm": 0.6862999200820923, + "kl": 0.04350169296376407, + "learning_rate": 4.835680751173709e-06, + "loss": 0.0017400771379470826, + "num_tokens": 81479.0, + "reward": 0.5159000098705292, + "reward_std": 0.17465537190437316, + "rewards/reward_fn/mean": 0.5158999919891357, + "rewards/reward_fn/std": 0.16455023884773254, "step": 20 }, { @@ -123,25 +123,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 27.25, + "completion_length": 28.0, "completions/clipped_ratio": 0.0, - "completions/max_length": 45.6, - "completions/max_terminated_length": 45.6, - "completions/mean_length": 27.25, - "completions/mean_terminated_length": 27.25, - "completions/min_length": 18.8, - "completions/min_terminated_length": 18.8, - "epoch": 0.11261261261261261, + "completions/max_length": 40.6, + "completions/max_terminated_length": 40.6, + "completions/mean_length": 28.0, + "completions/mean_terminated_length": 28.0, + "completions/min_length": 22.6, + "completions/min_terminated_length": 22.6, + "epoch": 0.1111111111111111, "frac_reward_zero_std": 0.0, - "grad_norm": 1.3132057189941406, - "kl": 0.003781939600594342, - "learning_rate": 4.714285714285715e-06, - "loss": 0.00015127062797546387, - "num_tokens": 103596.0, - "reward": 0.4680500030517578, - "reward_std": 0.17189764976501465, - "rewards/reward_fn/mean": 0.46804999113082885, - "rewards/reward_fn/std": 0.17421672195196153, + "grad_norm": 1.7516958713531494, + "kl": 0.05824573401478119, + "learning_rate": 4.71830985915493e-06, + "loss": 0.002329717576503754, + "num_tokens": 102833.0, + "reward": 0.5547999978065491, + "reward_std": 0.18328206837177277, + "rewards/reward_fn/mean": 0.5547999858856201, + "rewards/reward_fn/std": 0.1974634051322937, "step": 25 }, { @@ -150,25 +150,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 25.225, + "completion_length": 28.675, "completions/clipped_ratio": 0.0, - "completions/max_length": 34.6, - "completions/max_terminated_length": 34.6, - "completions/mean_length": 25.225, - "completions/mean_terminated_length": 25.225, - "completions/min_length": 19.2, - "completions/min_terminated_length": 19.2, - "epoch": 0.13513513513513514, + "completions/max_length": 37.4, + "completions/max_terminated_length": 37.4, + "completions/mean_length": 28.675, + "completions/mean_terminated_length": 28.675, + "completions/min_length": 22.2, + "completions/min_terminated_length": 22.2, + "epoch": 0.13333333333333333, "frac_reward_zero_std": 0.0, - "grad_norm": 1.5750545263290405, - "kl": 0.006673775642411783, - "learning_rate": 4.595238095238095e-06, - "loss": 0.0002669498324394226, - "num_tokens": 123651.0, - "reward": 0.48049998879432676, - "reward_std": 0.1656044065952301, - "rewards/reward_fn/mean": 0.48049998879432676, - "rewards/reward_fn/std": 0.13908967524766921, + "grad_norm": 1.5611271858215332, + "kl": 0.09752920938190072, + "learning_rate": 4.60093896713615e-06, + "loss": 0.0039011374115943908, + "num_tokens": 123876.0, + "reward": 0.5165499806404114, + "reward_std": 0.16765501499176025, + "rewards/reward_fn/mean": 0.5165499806404114, + "rewards/reward_fn/std": 0.16748485416173936, "step": 30 }, { @@ -177,25 +177,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 28.775, - "completions/clipped_ratio": 0.025, - "completions/max_length": 54.8, - "completions/max_terminated_length": 41.8, - "completions/mean_length": 28.775, - "completions/mean_terminated_length": 26.967857360839844, - "completions/min_length": 15.4, - "completions/min_terminated_length": 15.4, - "epoch": 0.15765765765765766, + "completion_length": 27.475, + "completions/clipped_ratio": 0.0, + "completions/max_length": 36.8, + "completions/max_terminated_length": 36.8, + "completions/mean_length": 27.475, + "completions/mean_terminated_length": 27.475, + "completions/min_length": 20.6, + "completions/min_terminated_length": 20.6, + "epoch": 0.15555555555555556, "frac_reward_zero_std": 0.0, - "grad_norm": 1.1105402708053589, - "kl": 0.005106111426721327, - "learning_rate": 4.476190476190477e-06, - "loss": 0.0002042025327682495, - "num_tokens": 144316.0, - "reward": 0.4778999924659729, - "reward_std": 0.17012988924980163, - "rewards/reward_fn/mean": 0.47790001034736634, - "rewards/reward_fn/std": 0.14147266000509262, + "grad_norm": 0.9550014138221741, + "kl": 0.08032013729680329, + "learning_rate": 4.483568075117371e-06, + "loss": 0.003212757408618927, + "num_tokens": 145019.0, + "reward": 0.5004499793052674, + "reward_std": 0.18391846716403962, + "rewards/reward_fn/mean": 0.5004500031471253, + "rewards/reward_fn/std": 0.16948954164981841, "step": 35 }, { @@ -204,25 +204,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 24.4, - "completions/clipped_ratio": 0.0, - "completions/max_length": 33.4, - "completions/max_terminated_length": 33.4, - "completions/mean_length": 24.4, - "completions/mean_terminated_length": 24.4, - "completions/min_length": 18.8, - "completions/min_terminated_length": 18.8, - "epoch": 0.18018018018018017, + "completion_length": 32.125, + "completions/clipped_ratio": 0.025, + "completions/max_length": 54.6, + "completions/max_terminated_length": 46.0, + "completions/mean_length": 32.125, + "completions/mean_terminated_length": 30.521428680419923, + "completions/min_length": 23.2, + "completions/min_terminated_length": 23.2, + "epoch": 0.17777777777777778, "frac_reward_zero_std": 0.0, - "grad_norm": 1.0676746368408203, - "kl": 0.006288998411764624, - "learning_rate": 4.357142857142857e-06, - "loss": 0.00025154203176498414, - "num_tokens": 164334.0, - "reward": 0.5014999985694886, - "reward_std": 0.15358359068632127, - "rewards/reward_fn/mean": 0.5014999985694886, - "rewards/reward_fn/std": 0.13824734836816788, + "grad_norm": 1.1203869581222534, + "kl": 0.08563535290304572, + "learning_rate": 4.3661971830985915e-06, + "loss": 0.003425435721874237, + "num_tokens": 164800.0, + "reward": 0.5472500026226044, + "reward_std": 0.19579786211252212, + "rewards/reward_fn/mean": 0.5472500085830688, + "rewards/reward_fn/std": 0.19857290089130403, "step": 40 }, { @@ -231,25 +231,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 27.35, + "completion_length": 27.875, "completions/clipped_ratio": 0.0, - "completions/max_length": 43.6, - "completions/max_terminated_length": 43.6, - "completions/mean_length": 27.35, - "completions/mean_terminated_length": 27.35, - "completions/min_length": 19.6, - "completions/min_terminated_length": 19.6, - "epoch": 0.20270270270270271, + "completions/max_length": 35.0, + "completions/max_terminated_length": 35.0, + "completions/mean_length": 27.875, + "completions/mean_terminated_length": 27.875, + "completions/min_length": 21.4, + "completions/min_terminated_length": 21.4, + "epoch": 0.2, "frac_reward_zero_std": 0.0, - "grad_norm": 1.0749404430389404, - "kl": 0.010629063473606948, - "learning_rate": 4.238095238095239e-06, - "loss": 0.0004251018166542053, - "num_tokens": 185140.0, - "reward": 0.524949985742569, - "reward_std": 0.14630039632320405, - "rewards/reward_fn/mean": 0.5249499917030335, - "rewards/reward_fn/std": 0.1460244983434677, + "grad_norm": 0.95042884349823, + "kl": 0.07295196709455923, + "learning_rate": 4.248826291079813e-06, + "loss": 0.0029180020093917845, + "num_tokens": 185761.0, + "reward": 0.5225999832153321, + "reward_std": 0.16489729881286622, + "rewards/reward_fn/mean": 0.5226000070571899, + "rewards/reward_fn/std": 0.16541497856378556, "step": 45 }, { @@ -258,25 +258,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 25.425, + "completion_length": 27.875, "completions/clipped_ratio": 0.0, "completions/max_length": 35.2, "completions/max_terminated_length": 35.2, - "completions/mean_length": 25.425, - "completions/mean_terminated_length": 25.425, - "completions/min_length": 17.6, - "completions/min_terminated_length": 17.6, - "epoch": 0.22522522522522523, + "completions/mean_length": 27.875, + "completions/mean_terminated_length": 27.875, + "completions/min_length": 21.4, + "completions/min_terminated_length": 21.4, + "epoch": 0.2222222222222222, "frac_reward_zero_std": 0.0, - "grad_norm": 1.4353065490722656, - "kl": 0.008290678875346203, - "learning_rate": 4.119047619047619e-06, - "loss": 0.0003315746784210205, - "num_tokens": 206569.0, - "reward": 0.4908999800682068, - "reward_std": 0.17055415213108063, - "rewards/reward_fn/mean": 0.49089999198913575, - "rewards/reward_fn/std": 0.154354290664196, + "grad_norm": 1.1037548780441284, + "kl": 0.08246680488809943, + "learning_rate": 4.131455399061034e-06, + "loss": 0.0032985761761665346, + "num_tokens": 206674.0, + "reward": 0.5245000004768372, + "reward_std": 0.1653215616941452, + "rewards/reward_fn/mean": 0.5245000004768372, + "rewards/reward_fn/std": 0.16877340227365495, "step": 50 }, { @@ -285,25 +285,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 27.7, - "completions/clipped_ratio": 0.025, - "completions/max_length": 50.6, - "completions/max_terminated_length": 37.2, - "completions/mean_length": 27.7, - "completions/mean_terminated_length": 25.91071434020996, - "completions/min_length": 18.2, - "completions/min_terminated_length": 18.2, - "epoch": 0.24774774774774774, + "completion_length": 29.225, + "completions/clipped_ratio": 0.0, + "completions/max_length": 47.4, + "completions/max_terminated_length": 47.4, + "completions/mean_length": 29.225, + "completions/mean_terminated_length": 29.225, + "completions/min_length": 20.2, + "completions/min_terminated_length": 20.2, + "epoch": 0.24444444444444444, "frac_reward_zero_std": 0.0, - "grad_norm": 0.8640509843826294, - "kl": 0.012456063569698018, - "learning_rate": 4.000000000000001e-06, - "loss": 0.0004982292652130127, - "num_tokens": 226097.0, - "reward": 0.4839499771595001, - "reward_std": 0.1590283066034317, - "rewards/reward_fn/mean": 0.48394999504089353, - "rewards/reward_fn/std": 0.1380786642432213, + "grad_norm": 0.8115681409835815, + "kl": 0.08297618771903217, + "learning_rate": 4.014084507042254e-06, + "loss": 0.0033189669251441956, + "num_tokens": 227351.0, + "reward": 0.5301500022411346, + "reward_std": 0.20612163245677947, + "rewards/reward_fn/mean": 0.5301500082015991, + "rewards/reward_fn/std": 0.19000594317913055, "step": 55 }, { @@ -312,25 +312,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 26.875, - "completions/clipped_ratio": 0.0, - "completions/max_length": 40.0, - "completions/max_terminated_length": 40.0, - "completions/mean_length": 26.875, - "completions/mean_terminated_length": 26.875, - "completions/min_length": 20.6, - "completions/min_terminated_length": 20.6, - "epoch": 0.2702702702702703, + "completion_length": 27.475, + "completions/clipped_ratio": 0.025, + "completions/max_length": 45.0, + "completions/max_terminated_length": 31.8, + "completions/mean_length": 27.475, + "completions/mean_terminated_length": 25.646428680419923, + "completions/min_length": 18.8, + "completions/min_terminated_length": 18.8, + "epoch": 0.26666666666666666, "frac_reward_zero_std": 0.0, - "grad_norm": 1.0529346466064453, - "kl": 0.007519814093393506, - "learning_rate": 3.880952380952381e-06, - "loss": 0.00030075013637542725, - "num_tokens": 246836.0, - "reward": 0.4914499819278717, - "reward_std": 0.1727461814880371, - "rewards/reward_fn/mean": 0.49144998788833616, - "rewards/reward_fn/std": 0.1568704679608345, + "grad_norm": 1.1722970008850098, + "kl": 0.06283304298995063, + "learning_rate": 3.896713615023475e-06, + "loss": 0.002513286471366882, + "num_tokens": 246464.0, + "reward": 0.513349997997284, + "reward_std": 0.18999958634376526, + "rewards/reward_fn/mean": 0.5133499920368194, + "rewards/reward_fn/std": 0.1748345360159874, "step": 60 }, { @@ -339,25 +339,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 27.825, - "completions/clipped_ratio": 0.025, - "completions/max_length": 51.2, - "completions/max_terminated_length": 38.0, - "completions/mean_length": 27.825, - "completions/mean_terminated_length": 26.03214302062988, - "completions/min_length": 17.6, - "completions/min_terminated_length": 17.6, - "epoch": 0.2927927927927928, + "completion_length": 26.075, + "completions/clipped_ratio": 0.0, + "completions/max_length": 34.2, + "completions/max_terminated_length": 34.2, + "completions/mean_length": 26.075, + "completions/mean_terminated_length": 26.075, + "completions/min_length": 19.4, + "completions/min_terminated_length": 19.4, + "epoch": 0.28888888888888886, "frac_reward_zero_std": 0.0, - "grad_norm": 1.149415373802185, - "kl": 0.00938094645498495, - "learning_rate": 3.761904761904762e-06, - "loss": 0.0003751114010810852, - "num_tokens": 266985.0, - "reward": 0.5003499746322632, - "reward_std": 0.16892780363559723, - "rewards/reward_fn/mean": 0.5003500044345855, - "rewards/reward_fn/std": 0.14663992822170258, + "grad_norm": 1.2825119495391846, + "kl": 0.07132846353342756, + "learning_rate": 3.779342723004695e-06, + "loss": 0.0028530970215797425, + "num_tokens": 267279.0, + "reward": 0.5383000016212464, + "reward_std": 0.20746512711048126, + "rewards/reward_fn/mean": 0.5383000195026397, + "rewards/reward_fn/std": 0.1835445523262024, "step": 65 }, { @@ -366,25 +366,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 27.0, + "completion_length": 27.925, "completions/clipped_ratio": 0.0, - "completions/max_length": 40.6, - "completions/max_terminated_length": 40.6, - "completions/mean_length": 27.0, - "completions/mean_terminated_length": 27.0, - "completions/min_length": 17.8, - "completions/min_terminated_length": 17.8, - "epoch": 0.3153153153153153, + "completions/max_length": 44.0, + "completions/max_terminated_length": 44.0, + "completions/mean_length": 27.925, + "completions/mean_terminated_length": 27.925, + "completions/min_length": 22.0, + "completions/min_terminated_length": 22.0, + "epoch": 0.3111111111111111, "frac_reward_zero_std": 0.0, - "grad_norm": 0.9893133640289307, - "kl": 0.010053556466664304, - "learning_rate": 3.642857142857143e-06, - "loss": 0.00040204524993896483, - "num_tokens": 288099.0, - "reward": 0.4705499827861786, - "reward_std": 0.16256383657455445, - "rewards/reward_fn/mean": 0.470550000667572, - "rewards/reward_fn/std": 0.13094386458396912, + "grad_norm": 0.5832120776176453, + "kl": 0.05197672065114602, + "learning_rate": 3.6619718309859158e-06, + "loss": 0.0020790368318557738, + "num_tokens": 287984.0, + "reward": 0.5424999892711639, + "reward_std": 0.18002938330173493, + "rewards/reward_fn/mean": 0.5424999952316284, + "rewards/reward_fn/std": 0.18769851326942444, "step": 70 }, { @@ -393,25 +393,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 25.75, - "completions/clipped_ratio": 0.0, - "completions/max_length": 34.8, - "completions/max_terminated_length": 34.8, - "completions/mean_length": 25.75, - "completions/mean_terminated_length": 25.75, - "completions/min_length": 19.8, - "completions/min_terminated_length": 19.8, - "epoch": 0.33783783783783783, + "completion_length": 27.6, + "completions/clipped_ratio": 0.025, + "completions/max_length": 44.6, + "completions/max_terminated_length": 30.6, + "completions/mean_length": 27.6, + "completions/mean_terminated_length": 25.807143020629884, + "completions/min_length": 22.2, + "completions/min_terminated_length": 22.2, + "epoch": 0.3333333333333333, "frac_reward_zero_std": 0.0, - "grad_norm": 1.063340187072754, - "kl": 0.008550376921630232, - "learning_rate": 3.523809523809524e-06, - "loss": 0.00034204572439193725, - "num_tokens": 308531.0, - "reward": 0.5000500082969666, - "reward_std": 0.16242243051528932, - "rewards/reward_fn/mean": 0.5000500082969666, - "rewards/reward_fn/std": 0.14417539685964584, + "grad_norm": 0.6677760481834412, + "kl": 0.03671608620206825, + "learning_rate": 3.5446009389671364e-06, + "loss": 0.0014685407280921937, + "num_tokens": 307800.0, + "reward": 0.5715999841690064, + "reward_std": 0.18639334440231323, + "rewards/reward_fn/mean": 0.5715999960899353, + "rewards/reward_fn/std": 0.19429495334625244, "step": 75 }, { @@ -420,25 +420,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 27.025, + "completion_length": 25.6, "completions/clipped_ratio": 0.0, - "completions/max_length": 40.0, - "completions/max_terminated_length": 40.0, - "completions/mean_length": 27.025, - "completions/mean_terminated_length": 27.025, - "completions/min_length": 19.4, - "completions/min_terminated_length": 19.4, - "epoch": 0.36036036036036034, + "completions/max_length": 30.2, + "completions/max_terminated_length": 30.2, + "completions/mean_length": 25.6, + "completions/mean_terminated_length": 25.6, + "completions/min_length": 21.4, + "completions/min_terminated_length": 21.4, + "epoch": 0.35555555555555557, "frac_reward_zero_std": 0.0, - "grad_norm": 1.2858343124389648, - "kl": 0.019255010827328077, - "learning_rate": 3.404761904761905e-06, - "loss": 0.0007702425122261047, - "num_tokens": 328874.0, - "reward": 0.5261500000953674, - "reward_std": 0.11886464804410934, - "rewards/reward_fn/mean": 0.5261499881744385, - "rewards/reward_fn/std": 0.12691220343112947, + "grad_norm": 0.7261527180671692, + "kl": 0.05616153636947274, + "learning_rate": 3.427230046948357e-06, + "loss": 0.0022463813424110413, + "num_tokens": 327794.0, + "reward": 0.5464499950408935, + "reward_std": 0.18264567852020264, + "rewards/reward_fn/mean": 0.5464499950408935, + "rewards/reward_fn/std": 0.17480863779783248, "step": 80 }, { @@ -447,25 +447,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 25.475, + "completion_length": 24.075, "completions/clipped_ratio": 0.0, - "completions/max_length": 33.0, - "completions/max_terminated_length": 33.0, - "completions/mean_length": 25.475, - "completions/mean_terminated_length": 25.475, - "completions/min_length": 19.0, - "completions/min_terminated_length": 19.0, - "epoch": 0.38288288288288286, + "completions/max_length": 29.6, + "completions/max_terminated_length": 29.6, + "completions/mean_length": 24.075, + "completions/mean_terminated_length": 24.075, + "completions/min_length": 16.6, + "completions/min_terminated_length": 16.6, + "epoch": 0.37777777777777777, "frac_reward_zero_std": 0.0, - "grad_norm": 1.118030309677124, - "kl": 0.01090274843845691, - "learning_rate": 3.285714285714286e-06, - "loss": 0.00043606162071228025, - "num_tokens": 348373.0, - "reward": 0.49950000643730164, - "reward_std": 0.15655344128608703, - "rewards/reward_fn/mean": 0.49950000643730164, - "rewards/reward_fn/std": 0.14320258051156998, + "grad_norm": 1.1086981296539307, + "kl": 0.07743949705036357, + "learning_rate": 3.3098591549295777e-06, + "loss": 0.0030974715948104857, + "num_tokens": 348233.0, + "reward": 0.5536999821662902, + "reward_std": 0.18073648810386658, + "rewards/reward_fn/mean": 0.5536999821662902, + "rewards/reward_fn/std": 0.1820658951997757, "step": 85 }, { @@ -474,25 +474,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 27.9, - "completions/clipped_ratio": 0.05, - "completions/max_length": 55.4, - "completions/max_terminated_length": 31.6, - "completions/mean_length": 27.9, - "completions/mean_terminated_length": 24.28571472167969, - "completions/min_length": 18.0, - "completions/min_terminated_length": 18.0, - "epoch": 0.40540540540540543, + "completion_length": 25.375, + "completions/clipped_ratio": 0.0, + "completions/max_length": 34.4, + "completions/max_terminated_length": 34.4, + "completions/mean_length": 25.375, + "completions/mean_terminated_length": 25.375, + "completions/min_length": 18.6, + "completions/min_terminated_length": 18.6, + "epoch": 0.4, "frac_reward_zero_std": 0.0, - "grad_norm": 0.6538166403770447, - "kl": 0.008556566922561615, - "learning_rate": 3.1666666666666667e-06, - "loss": 0.0003422081470489502, - "num_tokens": 368815.0, - "reward": 0.5215500056743622, - "reward_std": 0.16581654250621797, - "rewards/reward_fn/mean": 0.5215500056743622, - "rewards/reward_fn/std": 0.15472394227981567, + "grad_norm": 1.1903071403503418, + "kl": 0.058934826811309904, + "learning_rate": 3.1924882629107983e-06, + "loss": 0.002357317507266998, + "num_tokens": 367732.0, + "reward": 0.5361499905586242, + "reward_std": 0.179675829410553, + "rewards/reward_fn/mean": 0.5361500144004822, + "rewards/reward_fn/std": 0.18574500381946563, "step": 90 }, { @@ -501,25 +501,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 25.75, + "completion_length": 26.825, "completions/clipped_ratio": 0.0, - "completions/max_length": 32.4, - "completions/max_terminated_length": 32.4, - "completions/mean_length": 25.75, - "completions/mean_terminated_length": 25.75, + "completions/max_length": 37.2, + "completions/max_terminated_length": 37.2, + "completions/mean_length": 26.825, + "completions/mean_terminated_length": 26.825, "completions/min_length": 20.8, "completions/min_terminated_length": 20.8, - "epoch": 0.42792792792792794, + "epoch": 0.4222222222222222, "frac_reward_zero_std": 0.0, - "grad_norm": 1.180251955986023, - "kl": 0.014757526386529208, - "learning_rate": 3.047619047619048e-06, - "loss": 0.0005902737379074096, - "num_tokens": 388761.0, - "reward": 0.514000016450882, - "reward_std": 0.1465125188231468, - "rewards/reward_fn/mean": 0.5139999985694885, - "rewards/reward_fn/std": 0.14701166301965712, + "grad_norm": 1.0952492952346802, + "kl": 0.040232469444163144, + "learning_rate": 3.075117370892019e-06, + "loss": 0.0016092658042907715, + "num_tokens": 387075.0, + "reward": 0.5278500020503998, + "reward_std": 0.1932522773742676, + "rewards/reward_fn/mean": 0.5278499901294709, + "rewards/reward_fn/std": 0.1796583503484726, "step": 95 }, { @@ -528,25 +528,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 26.35, - "completions/clipped_ratio": 0.025, - "completions/max_length": 46.6, - "completions/max_terminated_length": 32.4, - "completions/mean_length": 26.35, - "completions/mean_terminated_length": 24.517857360839844, - "completions/min_length": 17.8, - "completions/min_terminated_length": 17.8, - "epoch": 0.45045045045045046, + "completion_length": 26.125, + "completions/clipped_ratio": 0.0, + "completions/max_length": 35.6, + "completions/max_terminated_length": 35.6, + "completions/mean_length": 26.125, + "completions/mean_terminated_length": 26.125, + "completions/min_length": 22.6, + "completions/min_terminated_length": 22.6, + "epoch": 0.4444444444444444, "frac_reward_zero_std": 0.0, - "grad_norm": 0.8863919973373413, - "kl": 0.017311154678463937, - "learning_rate": 2.928571428571429e-06, - "loss": 0.0006924748420715332, - "num_tokens": 408309.0, - "reward": 0.5527999818325042, - "reward_std": 0.1593818634748459, - "rewards/reward_fn/mean": 0.5527999699115753, - "rewards/reward_fn/std": 0.16823574155569077, + "grad_norm": 0.8072858452796936, + "kl": 0.06188266044482589, + "learning_rate": 2.9577464788732396e-06, + "loss": 0.0024752289056777952, + "num_tokens": 406884.0, + "reward": 0.5849499881267548, + "reward_std": 0.20499025285243988, + "rewards/reward_fn/mean": 0.5849500000476837, + "rewards/reward_fn/std": 0.2056175708770752, "step": 100 }, { @@ -555,25 +555,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 28.675, - "completions/clipped_ratio": 0.05, - "completions/max_length": 45.6, - "completions/max_terminated_length": 33.2, - "completions/mean_length": 28.675, - "completions/mean_terminated_length": 25.291666793823243, - "completions/min_length": 20.6, - "completions/min_terminated_length": 20.6, - "epoch": 0.47297297297297297, + "completion_length": 26.45, + "completions/clipped_ratio": 0.025, + "completions/max_length": 43.2, + "completions/max_terminated_length": 29.0, + "completions/mean_length": 26.45, + "completions/mean_terminated_length": 24.639286041259766, + "completions/min_length": 20.4, + "completions/min_terminated_length": 20.4, + "epoch": 0.4666666666666667, "frac_reward_zero_std": 0.0, - "grad_norm": 0.7957925200462341, - "kl": 0.021478149453469087, - "learning_rate": 2.8095238095238096e-06, - "loss": 0.0008591204881668091, - "num_tokens": 429872.0, - "reward": 0.5294999718666077, - "reward_std": 0.15301789939403534, - "rewards/reward_fn/mean": 0.5294999837875366, - "rewards/reward_fn/std": 0.15534335970878602, + "grad_norm": 0.9750375747680664, + "kl": 0.0710214663646184, + "learning_rate": 2.8403755868544603e-06, + "loss": 0.0028407976031303407, + "num_tokens": 427518.0, + "reward": 0.5496499896049499, + "reward_std": 0.16298811435699462, + "rewards/reward_fn/mean": 0.5496499896049499, + "rewards/reward_fn/std": 0.15672676265239716, "step": 105 }, { @@ -582,25 +582,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 27.325, + "completion_length": 25.925, "completions/clipped_ratio": 0.0, - "completions/max_length": 47.2, - "completions/max_terminated_length": 47.2, - "completions/mean_length": 27.325, - "completions/mean_terminated_length": 27.325, - "completions/min_length": 19.2, - "completions/min_terminated_length": 19.2, - "epoch": 0.4954954954954955, + "completions/max_length": 31.8, + "completions/max_terminated_length": 31.8, + "completions/mean_length": 25.925, + "completions/mean_terminated_length": 25.925, + "completions/min_length": 21.6, + "completions/min_terminated_length": 21.6, + "epoch": 0.4888888888888889, "frac_reward_zero_std": 0.0, - "grad_norm": 1.187671184539795, - "kl": 0.01666262859798735, - "learning_rate": 2.6904761904761906e-06, - "loss": 0.0006665512919425964, - "num_tokens": 450229.0, - "reward": 0.5364000022411346, - "reward_std": 0.16447303295135499, - "rewards/reward_fn/mean": 0.5364000022411346, - "rewards/reward_fn/std": 0.17169796973466872, + "grad_norm": 1.0184260606765747, + "kl": 0.0396142341895029, + "learning_rate": 2.723004694835681e-06, + "loss": 0.0015845373272895813, + "num_tokens": 447807.0, + "reward": 0.5718499898910523, + "reward_std": 0.20725298821926116, + "rewards/reward_fn/mean": 0.5718500018119812, + "rewards/reward_fn/std": 0.214348441362381, "step": 110 }, { @@ -609,25 +609,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 27.6, - "completions/clipped_ratio": 0.025, - "completions/max_length": 46.6, - "completions/max_terminated_length": 37.2, - "completions/mean_length": 27.6, - "completions/mean_terminated_length": 25.853571701049805, - "completions/min_length": 18.8, - "completions/min_terminated_length": 18.8, - "epoch": 0.5180180180180181, + "completion_length": 26.225, + "completions/clipped_ratio": 0.0, + "completions/max_length": 31.2, + "completions/max_terminated_length": 31.2, + "completions/mean_length": 26.225, + "completions/mean_terminated_length": 26.225, + "completions/min_length": 22.4, + "completions/min_terminated_length": 22.4, + "epoch": 0.5111111111111111, "frac_reward_zero_std": 0.0, - "grad_norm": 1.3785444498062134, - "kl": 0.015608730388339608, - "learning_rate": 2.571428571428571e-06, - "loss": 0.0006243243813514709, - "num_tokens": 470911.0, - "reward": 0.5131499886512756, - "reward_std": 0.15916974246501922, - "rewards/reward_fn/mean": 0.513150018453598, - "rewards/reward_fn/std": 0.15986726433038712, + "grad_norm": 0.8158187866210938, + "kl": 0.048210305260727185, + "learning_rate": 2.6056338028169015e-06, + "loss": 0.0019283831119537354, + "num_tokens": 468650.0, + "reward": 0.5343500077724457, + "reward_std": 0.1919794887304306, + "rewards/reward_fn/mean": 0.5343500018119812, + "rewards/reward_fn/std": 0.17689327299594879, "step": 115 }, { @@ -636,25 +636,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 26.725, - "completions/clipped_ratio": 0.025, - "completions/max_length": 43.8, - "completions/max_terminated_length": 30.4, - "completions/mean_length": 26.725, - "completions/mean_terminated_length": 24.953571701049803, - "completions/min_length": 20.4, - "completions/min_terminated_length": 20.4, - "epoch": 0.5405405405405406, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.8001337647438049, - "kl": 0.016426509176380933, - "learning_rate": 2.4523809523809526e-06, - "loss": 0.0006570681929588318, - "num_tokens": 490616.0, - "reward": 0.5314500093460083, - "reward_std": 0.16807928085327148, - "rewards/reward_fn/mean": 0.5314499974250794, - "rewards/reward_fn/std": 0.16840155124664308, + "completion_length": 25.875, + "completions/clipped_ratio": 0.0, + "completions/max_length": 30.0, + "completions/max_terminated_length": 30.0, + "completions/mean_length": 25.875, + "completions/mean_terminated_length": 25.875, + "completions/min_length": 24.2, + "completions/min_terminated_length": 24.2, + "epoch": 0.5333333333333333, + "frac_reward_zero_std": 0.05, + "grad_norm": 0.8932302594184875, + "kl": 0.05402324852766469, + "learning_rate": 2.488262910798122e-06, + "loss": 0.0021608427166938783, + "num_tokens": 489051.0, + "reward": 0.6121499896049499, + "reward_std": 0.16680648624897004, + "rewards/reward_fn/mean": 0.612150001525879, + "rewards/reward_fn/std": 0.18321824073791504, "step": 120 }, { @@ -663,25 +663,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 25.8, - "completions/clipped_ratio": 0.0, - "completions/max_length": 39.8, - "completions/max_terminated_length": 39.8, - "completions/mean_length": 25.8, - "completions/mean_terminated_length": 25.8, - "completions/min_length": 17.6, - "completions/min_terminated_length": 17.6, - "epoch": 0.5630630630630631, + "completion_length": 27.3, + "completions/clipped_ratio": 0.025, + "completions/max_length": 42.6, + "completions/max_terminated_length": 29.8, + "completions/mean_length": 27.3, + "completions/mean_terminated_length": 25.514286041259766, + "completions/min_length": 21.4, + "completions/min_terminated_length": 21.4, + "epoch": 0.5555555555555556, "frac_reward_zero_std": 0.0, - "grad_norm": 1.0382933616638184, - "kl": 0.016185989990481174, - "learning_rate": 2.3333333333333336e-06, - "loss": 0.000647495687007904, - "num_tokens": 509092.0, - "reward": 0.5144500017166138, - "reward_std": 0.15323003232479096, - "rewards/reward_fn/mean": 0.5144500017166138, - "rewards/reward_fn/std": 0.13980331867933274, + "grad_norm": 0.922471821308136, + "kl": 0.04857689954806119, + "learning_rate": 2.370892018779343e-06, + "loss": 0.0019433587789535523, + "num_tokens": 508231.0, + "reward": 0.5884499907493591, + "reward_std": 0.17119054943323136, + "rewards/reward_fn/mean": 0.588450014591217, + "rewards/reward_fn/std": 0.18389662504196166, "step": 125 }, { @@ -690,25 +690,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 26.275, + "completion_length": 26.05, "completions/clipped_ratio": 0.0, - "completions/max_length": 36.4, - "completions/max_terminated_length": 36.4, - "completions/mean_length": 26.275, - "completions/mean_terminated_length": 26.275, - "completions/min_length": 19.6, - "completions/min_terminated_length": 19.6, - "epoch": 0.5855855855855856, + "completions/max_length": 31.0, + "completions/max_terminated_length": 31.0, + "completions/mean_length": 26.05, + "completions/mean_terminated_length": 26.05, + "completions/min_length": 22.0, + "completions/min_terminated_length": 22.0, + "epoch": 0.5777777777777777, "frac_reward_zero_std": 0.0, - "grad_norm": 1.3675200939178467, - "kl": 0.01404028357937932, - "learning_rate": 2.2142857142857146e-06, - "loss": 0.0005615666508674621, - "num_tokens": 531083.0, - "reward": 0.5205999791622162, - "reward_std": 0.1824335426092148, - "rewards/reward_fn/mean": 0.520600003004074, - "rewards/reward_fn/std": 0.16652330607175828, + "grad_norm": 0.7222557067871094, + "kl": 0.06997429557377473, + "learning_rate": 2.2535211267605635e-06, + "loss": 0.0027989834547042848, + "num_tokens": 528729.0, + "reward": 0.5927999973297119, + "reward_std": 0.22429427206516267, + "rewards/reward_fn/mean": 0.5928000092506409, + "rewards/reward_fn/std": 0.21314262747764587, "step": 130 }, { @@ -717,25 +717,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 27.025, + "completion_length": 25.925, "completions/clipped_ratio": 0.0, - "completions/max_length": 41.0, - "completions/max_terminated_length": 41.0, - "completions/mean_length": 27.025, - "completions/mean_terminated_length": 27.025, - "completions/min_length": 20.2, - "completions/min_terminated_length": 20.2, - "epoch": 0.6081081081081081, + "completions/max_length": 35.6, + "completions/max_terminated_length": 35.6, + "completions/mean_length": 25.925, + "completions/mean_terminated_length": 25.925, + "completions/min_length": 18.8, + "completions/min_terminated_length": 18.8, + "epoch": 0.6, "frac_reward_zero_std": 0.0, - "grad_norm": 1.5435948371887207, - "kl": 0.01980702848522924, - "learning_rate": 2.0952380952380955e-06, - "loss": 0.0007923290133476258, - "num_tokens": 551052.0, - "reward": 0.5506500005722046, - "reward_std": 0.15252292901277542, - "rewards/reward_fn/mean": 0.5506499886512757, - "rewards/reward_fn/std": 0.16045962125062943, + "grad_norm": 1.2586678266525269, + "kl": 0.08584307442652062, + "learning_rate": 2.136150234741784e-06, + "loss": 0.0034336388111114503, + "num_tokens": 548748.0, + "reward": 0.5738499760627747, + "reward_std": 0.17571603059768676, + "rewards/reward_fn/mean": 0.5738499999046326, + "rewards/reward_fn/std": 0.19411510229110718, "step": 135 }, { @@ -744,25 +744,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 24.875, + "completion_length": 25.9, "completions/clipped_ratio": 0.0, - "completions/max_length": 30.0, - "completions/max_terminated_length": 30.0, - "completions/mean_length": 24.875, - "completions/mean_terminated_length": 24.875, - "completions/min_length": 19.8, - "completions/min_terminated_length": 19.8, - "epoch": 0.6306306306306306, + "completions/max_length": 29.4, + "completions/max_terminated_length": 29.4, + "completions/mean_length": 25.9, + "completions/mean_terminated_length": 25.9, + "completions/min_length": 23.4, + "completions/min_terminated_length": 23.4, + "epoch": 0.6222222222222222, "frac_reward_zero_std": 0.0, - "grad_norm": 1.0660195350646973, - "kl": 0.014228896767599508, - "learning_rate": 1.976190476190476e-06, - "loss": 0.0005690708756446838, - "num_tokens": 570993.0, - "reward": 0.5141499876976013, - "reward_std": 0.1573312520980835, - "rewards/reward_fn/mean": 0.5141499936580658, - "rewards/reward_fn/std": 0.14991891831159593, + "grad_norm": 1.0420219898223877, + "kl": 0.06210445412434638, + "learning_rate": 2.0187793427230047e-06, + "loss": 0.0024841248989105223, + "num_tokens": 569094.0, + "reward": 0.6095999956130982, + "reward_std": 0.2001112163066864, + "rewards/reward_fn/mean": 0.6095999956130982, + "rewards/reward_fn/std": 0.18984024077653885, "step": 140 }, { @@ -771,25 +771,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 24.775, - "completions/clipped_ratio": 0.0, - "completions/max_length": 35.4, - "completions/max_terminated_length": 35.4, - "completions/mean_length": 24.775, - "completions/mean_terminated_length": 24.775, - "completions/min_length": 17.0, - "completions/min_terminated_length": 17.0, - "epoch": 0.6531531531531531, + "completion_length": 27.85, + "completions/clipped_ratio": 0.025, + "completions/max_length": 46.0, + "completions/max_terminated_length": 34.8, + "completions/mean_length": 27.85, + "completions/mean_terminated_length": 26.164286041259764, + "completions/min_length": 21.2, + "completions/min_terminated_length": 21.2, + "epoch": 0.6444444444444445, "frac_reward_zero_std": 0.0, - "grad_norm": 0.9134359359741211, - "kl": 0.015506639698287472, - "learning_rate": 1.8571428571428573e-06, - "loss": 0.0006200879812240601, - "num_tokens": 591118.0, - "reward": 0.5515999913215637, - "reward_std": 0.1493409514427185, - "rewards/reward_fn/mean": 0.5516000032424927, - "rewards/reward_fn/std": 0.15984065383672713, + "grad_norm": 0.9585386514663696, + "kl": 0.04591481959214434, + "learning_rate": 1.9014084507042254e-06, + "loss": 0.0018365621566772462, + "num_tokens": 589932.0, + "reward": 0.5361499905586242, + "reward_std": 0.20909147560596467, + "rewards/reward_fn/mean": 0.5361500024795532, + "rewards/reward_fn/std": 0.1902428910136223, "step": 145 }, { @@ -798,31 +798,31 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 25.45, + "completion_length": 25.25, "completions/clipped_ratio": 0.0, - "completions/max_length": 34.6, - "completions/max_terminated_length": 34.6, - "completions/mean_length": 25.45, - "completions/mean_terminated_length": 25.45, - "completions/min_length": 20.6, - "completions/min_terminated_length": 20.6, - "epoch": 0.6756756756756757, + "completions/max_length": 29.6, + "completions/max_terminated_length": 29.6, + "completions/mean_length": 25.25, + "completions/mean_terminated_length": 25.25, + "completions/min_length": 21.6, + "completions/min_terminated_length": 21.6, + "epoch": 0.6666666666666666, "frac_reward_zero_std": 0.0, - "grad_norm": 1.011871099472046, - "kl": 0.018102088455634657, - "learning_rate": 1.738095238095238e-06, - "loss": 0.0007240131497383117, - "num_tokens": 611492.0, - "reward": 0.5036499917507171, - "reward_std": 0.14262343645095826, - "rewards/reward_fn/mean": 0.5036500096321106, - "rewards/reward_fn/std": 0.14135744124650956, + "grad_norm": 0.45393237471580505, + "kl": 0.04076897802297026, + "learning_rate": 1.784037558685446e-06, + "loss": 0.001630684733390808, + "num_tokens": 610754.0, + "reward": 0.5666499972343445, + "reward_std": 0.17826161682605743, + "rewards/reward_fn/mean": 0.5666500210762024, + "rewards/reward_fn/std": 0.1894826263189316, "step": 150 } ], "logging_steps": 5, - "max_steps": 222, - "num_input_tokens_seen": 611492, + "max_steps": 225, + "num_input_tokens_seen": 610754, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { diff --git a/results/phase1/checkpoint-200/README.md b/results/phase1/checkpoint-200/README.md index 002e352e3f4655c9f00611b74eefa7b221963ec8..93a54877d7d12fb86dbb10111c6d07e97a31b3ea 100644 --- a/results/phase1/checkpoint-200/README.md +++ b/results/phase1/checkpoint-200/README.md @@ -6,6 +6,7 @@ tags: - base_model:adapter:unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit - grpo - lora +- sft - transformers - trl - unsloth diff --git a/results/phase1/checkpoint-200/adapter_config.json b/results/phase1/checkpoint-200/adapter_config.json index 5e64881c1f2a3150228a608fb85ae9d9b6278fb9..d3cb3463c7f8c805e8cd8c25e64f3af87d6a00f3 100644 --- a/results/phase1/checkpoint-200/adapter_config.json +++ b/results/phase1/checkpoint-200/adapter_config.json @@ -33,13 +33,13 @@ "rank_pattern": {}, "revision": null, "target_modules": [ + "gate_proj", + "q_proj", "k_proj", - "up_proj", + "v_proj", "down_proj", "o_proj", - "v_proj", - "gate_proj", - "q_proj" + "up_proj" ], "target_parameters": null, "task_type": "CAUSAL_LM", diff --git a/results/phase1/checkpoint-200/adapter_model.safetensors b/results/phase1/checkpoint-200/adapter_model.safetensors index ab764a3bc4443a3aac3aee7d5526e104880c6eaf..acf50ddbc8f9888f9c3bf1ac2cd7ca7c3b667a95 100644 --- a/results/phase1/checkpoint-200/adapter_model.safetensors +++ b/results/phase1/checkpoint-200/adapter_model.safetensors @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4c26ece2f3d9d3d4dad0b3f24c8a55189a45b259a581746379eb5042892d1ab9 +oid sha256:5e679fe7063df93e5693ca5c4b4078626e9664ca0d3c4b56177004f47e81eb4c size 73911112 diff --git a/results/phase1/checkpoint-200/optimizer.pt b/results/phase1/checkpoint-200/optimizer.pt index 6fd09b8b631f29c134de76ccb8a77af31d784866..eed0ab45eff212ea8b235536a89272f9973ea441 100644 --- a/results/phase1/checkpoint-200/optimizer.pt +++ b/results/phase1/checkpoint-200/optimizer.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:419e0e8af2f322b4b5a8c8665ec83a782bd5994d32068a47bde14d574cf26693 +oid sha256:a08e8876ad967f49ce5843759247ca1b2a750c2ebc688dd5d7354d16fcfceab9 size 37969669 diff --git a/results/phase1/checkpoint-200/rng_state.pth b/results/phase1/checkpoint-200/rng_state.pth index 87367205846637324517837b5b3240c6ba955ee3..6c00079d28c21e115dc74c415e5296fb3538671f 100644 --- a/results/phase1/checkpoint-200/rng_state.pth +++ b/results/phase1/checkpoint-200/rng_state.pth @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:31ac89641ae8e51a4d163ce36876b5f098ce283051a404ec657132b0ed089180 +oid sha256:25069db7710c4bef552b9b7fbd88f4d398b7354c1ed115dc6c7f865fcdb5a64c size 14645 diff --git a/results/phase1/checkpoint-200/scheduler.pt b/results/phase1/checkpoint-200/scheduler.pt index 2ad4a620cddb4d59ee91c21ca8561f80d13fb890..ad96483b7fec90b2b03f6095bd8e859e9fe317db 100644 --- a/results/phase1/checkpoint-200/scheduler.pt +++ b/results/phase1/checkpoint-200/scheduler.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4d518c8bde4bc6896434cd6cc11418da6ed7291dd8afe1cbd0a22b41b24de64f +oid sha256:2685d006b729d8986d8fc6e89b6e7f575ddbe4b9f764f7b9f7d433771599981c size 1465 diff --git a/results/phase1/checkpoint-200/trainer_state.json b/results/phase1/checkpoint-200/trainer_state.json index 3cd423e045fc382196ddc989e975cf37c5ce2f73..d9952d60854d959fe000ae93b7850e2626673836 100644 --- a/results/phase1/checkpoint-200/trainer_state.json +++ b/results/phase1/checkpoint-200/trainer_state.json @@ -2,7 +2,7 @@ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, - "epoch": 0.9009009009009009, + "epoch": 0.8888888888888888, "eval_steps": 500, "global_step": 200, "is_hyper_param_search": false, @@ -15,25 +15,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 29.05, - "completions/clipped_ratio": 0.05, - "completions/max_length": 44.2, - "completions/max_terminated_length": 31.0, - "completions/mean_length": 29.05, - "completions/mean_terminated_length": 25.583333587646486, - "completions/min_length": 21.4, - "completions/min_terminated_length": 21.4, - "epoch": 0.02252252252252252, - "frac_reward_zero_std": 0.0, - "grad_norm": 1.2577366828918457, - "kl": 7.414049605358741e-06, + "completion_length": 25.075, + "completions/clipped_ratio": 0.0, + "completions/max_length": 30.8, + "completions/max_terminated_length": 30.8, + "completions/mean_length": 25.075, + "completions/mean_terminated_length": 25.075, + "completions/min_length": 19.4, + "completions/min_terminated_length": 19.4, + "epoch": 0.022222222222222223, + "frac_reward_zero_std": 0.6, + "grad_norm": 1.3092279434204102, + "kl": 0.03290070100920275, "learning_rate": 1.6666666666666667e-06, - "loss": 2.8312206268310547e-07, - "num_tokens": 21378.0, - "reward": 0.4944999933242798, - "reward_std": 0.15089658200740813, - "rewards/reward_fn/mean": 0.49449999928474425, - "rewards/reward_fn/std": 0.12999328523874282, + "loss": 0.0013160213828086853, + "num_tokens": 20275.0, + "reward": 0.48624999523162843, + "reward_std": 0.05833630859851837, + "rewards/reward_fn/mean": 0.4862500011920929, + "rewards/reward_fn/std": 0.04605271518230438, "step": 5 }, { @@ -42,25 +42,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 29.175, + "completion_length": 25.925, "completions/clipped_ratio": 0.0, - "completions/max_length": 41.8, - "completions/max_terminated_length": 41.8, - "completions/mean_length": 29.175, - "completions/mean_terminated_length": 29.175, - "completions/min_length": 19.2, - "completions/min_terminated_length": 19.2, - "epoch": 0.04504504504504504, + "completions/max_length": 33.0, + "completions/max_terminated_length": 33.0, + "completions/mean_length": 25.925, + "completions/mean_terminated_length": 25.925, + "completions/min_length": 19.4, + "completions/min_terminated_length": 19.4, + "epoch": 0.044444444444444446, "frac_reward_zero_std": 0.0, - "grad_norm": 1.148110032081604, - "kl": 2.58529256058182e-05, + "grad_norm": 1.4204450845718384, + "kl": 0.040759827199508436, "learning_rate": 3.7500000000000005e-06, - "loss": 9.343028068542481e-07, - "num_tokens": 42709.0, - "reward": 0.48589999675750734, - "reward_std": 0.14184561967849732, - "rewards/reward_fn/mean": 0.4859000027179718, - "rewards/reward_fn/std": 0.12539554834365846, + "loss": 0.0016303554177284241, + "num_tokens": 41560.0, + "reward": 0.5283999860286712, + "reward_std": 0.17366542518138886, + "rewards/reward_fn/mean": 0.5283999919891358, + "rewards/reward_fn/std": 0.16809422075748442, "step": 10 }, { @@ -69,25 +69,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 28.825, + "completion_length": 27.45, "completions/clipped_ratio": 0.0, - "completions/max_length": 44.2, - "completions/max_terminated_length": 44.2, - "completions/mean_length": 28.825, - "completions/mean_terminated_length": 28.825, - "completions/min_length": 21.2, - "completions/min_terminated_length": 21.2, - "epoch": 0.06756756756756757, + "completions/max_length": 33.6, + "completions/max_terminated_length": 33.6, + "completions/mean_length": 27.45, + "completions/mean_terminated_length": 27.45, + "completions/min_length": 24.0, + "completions/min_terminated_length": 24.0, + "epoch": 0.06666666666666667, "frac_reward_zero_std": 0.0, - "grad_norm": 1.1520825624465942, - "kl": 0.0001522944080875277, - "learning_rate": 4.952380952380953e-06, - "loss": 6.105005741119385e-06, - "num_tokens": 63008.0, - "reward": 0.4894999861717224, - "reward_std": 0.15768481492996217, - "rewards/reward_fn/mean": 0.48950000405311583, - "rewards/reward_fn/std": 0.14185291826725005, + "grad_norm": 0.9515441656112671, + "kl": 0.049849181214813146, + "learning_rate": 4.953051643192488e-06, + "loss": 0.0019938603043556215, + "num_tokens": 62246.0, + "reward": 0.4975499987602234, + "reward_std": 0.1564827263355255, + "rewards/reward_fn/mean": 0.49755001068115234, + "rewards/reward_fn/std": 0.14618260115385057, "step": 15 }, { @@ -96,25 +96,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 28.25, + "completion_length": 26.525, "completions/clipped_ratio": 0.0, - "completions/max_length": 43.2, - "completions/max_terminated_length": 43.2, - "completions/mean_length": 28.25, - "completions/mean_terminated_length": 28.25, - "completions/min_length": 21.6, - "completions/min_terminated_length": 21.6, - "epoch": 0.09009009009009009, + "completions/max_length": 33.0, + "completions/max_terminated_length": 33.0, + "completions/mean_length": 26.525, + "completions/mean_terminated_length": 26.525, + "completions/min_length": 22.2, + "completions/min_terminated_length": 22.2, + "epoch": 0.08888888888888889, "frac_reward_zero_std": 0.0, - "grad_norm": 1.0063226222991943, - "kl": 0.0008983879492006963, - "learning_rate": 4.833333333333333e-06, - "loss": 3.593862056732178e-05, - "num_tokens": 82836.0, - "reward": 0.4881500005722046, - "reward_std": 0.14941166192293168, - "rewards/reward_fn/mean": 0.48814999461174013, - "rewards/reward_fn/std": 0.12791907638311387, + "grad_norm": 0.6862999200820923, + "kl": 0.04350169296376407, + "learning_rate": 4.835680751173709e-06, + "loss": 0.0017400771379470826, + "num_tokens": 81479.0, + "reward": 0.5159000098705292, + "reward_std": 0.17465537190437316, + "rewards/reward_fn/mean": 0.5158999919891357, + "rewards/reward_fn/std": 0.16455023884773254, "step": 20 }, { @@ -123,25 +123,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 27.25, + "completion_length": 28.0, "completions/clipped_ratio": 0.0, - "completions/max_length": 45.6, - "completions/max_terminated_length": 45.6, - "completions/mean_length": 27.25, - "completions/mean_terminated_length": 27.25, - "completions/min_length": 18.8, - "completions/min_terminated_length": 18.8, - "epoch": 0.11261261261261261, + "completions/max_length": 40.6, + "completions/max_terminated_length": 40.6, + "completions/mean_length": 28.0, + "completions/mean_terminated_length": 28.0, + "completions/min_length": 22.6, + "completions/min_terminated_length": 22.6, + "epoch": 0.1111111111111111, "frac_reward_zero_std": 0.0, - "grad_norm": 1.3132057189941406, - "kl": 0.003781939600594342, - "learning_rate": 4.714285714285715e-06, - "loss": 0.00015127062797546387, - "num_tokens": 103596.0, - "reward": 0.4680500030517578, - "reward_std": 0.17189764976501465, - "rewards/reward_fn/mean": 0.46804999113082885, - "rewards/reward_fn/std": 0.17421672195196153, + "grad_norm": 1.7516958713531494, + "kl": 0.05824573401478119, + "learning_rate": 4.71830985915493e-06, + "loss": 0.002329717576503754, + "num_tokens": 102833.0, + "reward": 0.5547999978065491, + "reward_std": 0.18328206837177277, + "rewards/reward_fn/mean": 0.5547999858856201, + "rewards/reward_fn/std": 0.1974634051322937, "step": 25 }, { @@ -150,25 +150,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 25.225, + "completion_length": 28.675, "completions/clipped_ratio": 0.0, - "completions/max_length": 34.6, - "completions/max_terminated_length": 34.6, - "completions/mean_length": 25.225, - "completions/mean_terminated_length": 25.225, - "completions/min_length": 19.2, - "completions/min_terminated_length": 19.2, - "epoch": 0.13513513513513514, + "completions/max_length": 37.4, + "completions/max_terminated_length": 37.4, + "completions/mean_length": 28.675, + "completions/mean_terminated_length": 28.675, + "completions/min_length": 22.2, + "completions/min_terminated_length": 22.2, + "epoch": 0.13333333333333333, "frac_reward_zero_std": 0.0, - "grad_norm": 1.5750545263290405, - "kl": 0.006673775642411783, - "learning_rate": 4.595238095238095e-06, - "loss": 0.0002669498324394226, - "num_tokens": 123651.0, - "reward": 0.48049998879432676, - "reward_std": 0.1656044065952301, - "rewards/reward_fn/mean": 0.48049998879432676, - "rewards/reward_fn/std": 0.13908967524766921, + "grad_norm": 1.5611271858215332, + "kl": 0.09752920938190072, + "learning_rate": 4.60093896713615e-06, + "loss": 0.0039011374115943908, + "num_tokens": 123876.0, + "reward": 0.5165499806404114, + "reward_std": 0.16765501499176025, + "rewards/reward_fn/mean": 0.5165499806404114, + "rewards/reward_fn/std": 0.16748485416173936, "step": 30 }, { @@ -177,25 +177,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 28.775, - "completions/clipped_ratio": 0.025, - "completions/max_length": 54.8, - "completions/max_terminated_length": 41.8, - "completions/mean_length": 28.775, - "completions/mean_terminated_length": 26.967857360839844, - "completions/min_length": 15.4, - "completions/min_terminated_length": 15.4, - "epoch": 0.15765765765765766, + "completion_length": 27.475, + "completions/clipped_ratio": 0.0, + "completions/max_length": 36.8, + "completions/max_terminated_length": 36.8, + "completions/mean_length": 27.475, + "completions/mean_terminated_length": 27.475, + "completions/min_length": 20.6, + "completions/min_terminated_length": 20.6, + "epoch": 0.15555555555555556, "frac_reward_zero_std": 0.0, - "grad_norm": 1.1105402708053589, - "kl": 0.005106111426721327, - "learning_rate": 4.476190476190477e-06, - "loss": 0.0002042025327682495, - "num_tokens": 144316.0, - "reward": 0.4778999924659729, - "reward_std": 0.17012988924980163, - "rewards/reward_fn/mean": 0.47790001034736634, - "rewards/reward_fn/std": 0.14147266000509262, + "grad_norm": 0.9550014138221741, + "kl": 0.08032013729680329, + "learning_rate": 4.483568075117371e-06, + "loss": 0.003212757408618927, + "num_tokens": 145019.0, + "reward": 0.5004499793052674, + "reward_std": 0.18391846716403962, + "rewards/reward_fn/mean": 0.5004500031471253, + "rewards/reward_fn/std": 0.16948954164981841, "step": 35 }, { @@ -204,25 +204,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 24.4, - "completions/clipped_ratio": 0.0, - "completions/max_length": 33.4, - "completions/max_terminated_length": 33.4, - "completions/mean_length": 24.4, - "completions/mean_terminated_length": 24.4, - "completions/min_length": 18.8, - "completions/min_terminated_length": 18.8, - "epoch": 0.18018018018018017, + "completion_length": 32.125, + "completions/clipped_ratio": 0.025, + "completions/max_length": 54.6, + "completions/max_terminated_length": 46.0, + "completions/mean_length": 32.125, + "completions/mean_terminated_length": 30.521428680419923, + "completions/min_length": 23.2, + "completions/min_terminated_length": 23.2, + "epoch": 0.17777777777777778, "frac_reward_zero_std": 0.0, - "grad_norm": 1.0676746368408203, - "kl": 0.006288998411764624, - "learning_rate": 4.357142857142857e-06, - "loss": 0.00025154203176498414, - "num_tokens": 164334.0, - "reward": 0.5014999985694886, - "reward_std": 0.15358359068632127, - "rewards/reward_fn/mean": 0.5014999985694886, - "rewards/reward_fn/std": 0.13824734836816788, + "grad_norm": 1.1203869581222534, + "kl": 0.08563535290304572, + "learning_rate": 4.3661971830985915e-06, + "loss": 0.003425435721874237, + "num_tokens": 164800.0, + "reward": 0.5472500026226044, + "reward_std": 0.19579786211252212, + "rewards/reward_fn/mean": 0.5472500085830688, + "rewards/reward_fn/std": 0.19857290089130403, "step": 40 }, { @@ -231,25 +231,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 27.35, + "completion_length": 27.875, "completions/clipped_ratio": 0.0, - "completions/max_length": 43.6, - "completions/max_terminated_length": 43.6, - "completions/mean_length": 27.35, - "completions/mean_terminated_length": 27.35, - "completions/min_length": 19.6, - "completions/min_terminated_length": 19.6, - "epoch": 0.20270270270270271, + "completions/max_length": 35.0, + "completions/max_terminated_length": 35.0, + "completions/mean_length": 27.875, + "completions/mean_terminated_length": 27.875, + "completions/min_length": 21.4, + "completions/min_terminated_length": 21.4, + "epoch": 0.2, "frac_reward_zero_std": 0.0, - "grad_norm": 1.0749404430389404, - "kl": 0.010629063473606948, - "learning_rate": 4.238095238095239e-06, - "loss": 0.0004251018166542053, - "num_tokens": 185140.0, - "reward": 0.524949985742569, - "reward_std": 0.14630039632320405, - "rewards/reward_fn/mean": 0.5249499917030335, - "rewards/reward_fn/std": 0.1460244983434677, + "grad_norm": 0.95042884349823, + "kl": 0.07295196709455923, + "learning_rate": 4.248826291079813e-06, + "loss": 0.0029180020093917845, + "num_tokens": 185761.0, + "reward": 0.5225999832153321, + "reward_std": 0.16489729881286622, + "rewards/reward_fn/mean": 0.5226000070571899, + "rewards/reward_fn/std": 0.16541497856378556, "step": 45 }, { @@ -258,25 +258,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 25.425, + "completion_length": 27.875, "completions/clipped_ratio": 0.0, "completions/max_length": 35.2, "completions/max_terminated_length": 35.2, - "completions/mean_length": 25.425, - "completions/mean_terminated_length": 25.425, - "completions/min_length": 17.6, - "completions/min_terminated_length": 17.6, - "epoch": 0.22522522522522523, + "completions/mean_length": 27.875, + "completions/mean_terminated_length": 27.875, + "completions/min_length": 21.4, + "completions/min_terminated_length": 21.4, + "epoch": 0.2222222222222222, "frac_reward_zero_std": 0.0, - "grad_norm": 1.4353065490722656, - "kl": 0.008290678875346203, - "learning_rate": 4.119047619047619e-06, - "loss": 0.0003315746784210205, - "num_tokens": 206569.0, - "reward": 0.4908999800682068, - "reward_std": 0.17055415213108063, - "rewards/reward_fn/mean": 0.49089999198913575, - "rewards/reward_fn/std": 0.154354290664196, + "grad_norm": 1.1037548780441284, + "kl": 0.08246680488809943, + "learning_rate": 4.131455399061034e-06, + "loss": 0.0032985761761665346, + "num_tokens": 206674.0, + "reward": 0.5245000004768372, + "reward_std": 0.1653215616941452, + "rewards/reward_fn/mean": 0.5245000004768372, + "rewards/reward_fn/std": 0.16877340227365495, "step": 50 }, { @@ -285,25 +285,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 27.7, - "completions/clipped_ratio": 0.025, - "completions/max_length": 50.6, - "completions/max_terminated_length": 37.2, - "completions/mean_length": 27.7, - "completions/mean_terminated_length": 25.91071434020996, - "completions/min_length": 18.2, - "completions/min_terminated_length": 18.2, - "epoch": 0.24774774774774774, + "completion_length": 29.225, + "completions/clipped_ratio": 0.0, + "completions/max_length": 47.4, + "completions/max_terminated_length": 47.4, + "completions/mean_length": 29.225, + "completions/mean_terminated_length": 29.225, + "completions/min_length": 20.2, + "completions/min_terminated_length": 20.2, + "epoch": 0.24444444444444444, "frac_reward_zero_std": 0.0, - "grad_norm": 0.8640509843826294, - "kl": 0.012456063569698018, - "learning_rate": 4.000000000000001e-06, - "loss": 0.0004982292652130127, - "num_tokens": 226097.0, - "reward": 0.4839499771595001, - "reward_std": 0.1590283066034317, - "rewards/reward_fn/mean": 0.48394999504089353, - "rewards/reward_fn/std": 0.1380786642432213, + "grad_norm": 0.8115681409835815, + "kl": 0.08297618771903217, + "learning_rate": 4.014084507042254e-06, + "loss": 0.0033189669251441956, + "num_tokens": 227351.0, + "reward": 0.5301500022411346, + "reward_std": 0.20612163245677947, + "rewards/reward_fn/mean": 0.5301500082015991, + "rewards/reward_fn/std": 0.19000594317913055, "step": 55 }, { @@ -312,25 +312,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 26.875, - "completions/clipped_ratio": 0.0, - "completions/max_length": 40.0, - "completions/max_terminated_length": 40.0, - "completions/mean_length": 26.875, - "completions/mean_terminated_length": 26.875, - "completions/min_length": 20.6, - "completions/min_terminated_length": 20.6, - "epoch": 0.2702702702702703, + "completion_length": 27.475, + "completions/clipped_ratio": 0.025, + "completions/max_length": 45.0, + "completions/max_terminated_length": 31.8, + "completions/mean_length": 27.475, + "completions/mean_terminated_length": 25.646428680419923, + "completions/min_length": 18.8, + "completions/min_terminated_length": 18.8, + "epoch": 0.26666666666666666, "frac_reward_zero_std": 0.0, - "grad_norm": 1.0529346466064453, - "kl": 0.007519814093393506, - "learning_rate": 3.880952380952381e-06, - "loss": 0.00030075013637542725, - "num_tokens": 246836.0, - "reward": 0.4914499819278717, - "reward_std": 0.1727461814880371, - "rewards/reward_fn/mean": 0.49144998788833616, - "rewards/reward_fn/std": 0.1568704679608345, + "grad_norm": 1.1722970008850098, + "kl": 0.06283304298995063, + "learning_rate": 3.896713615023475e-06, + "loss": 0.002513286471366882, + "num_tokens": 246464.0, + "reward": 0.513349997997284, + "reward_std": 0.18999958634376526, + "rewards/reward_fn/mean": 0.5133499920368194, + "rewards/reward_fn/std": 0.1748345360159874, "step": 60 }, { @@ -339,25 +339,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 27.825, - "completions/clipped_ratio": 0.025, - "completions/max_length": 51.2, - "completions/max_terminated_length": 38.0, - "completions/mean_length": 27.825, - "completions/mean_terminated_length": 26.03214302062988, - "completions/min_length": 17.6, - "completions/min_terminated_length": 17.6, - "epoch": 0.2927927927927928, + "completion_length": 26.075, + "completions/clipped_ratio": 0.0, + "completions/max_length": 34.2, + "completions/max_terminated_length": 34.2, + "completions/mean_length": 26.075, + "completions/mean_terminated_length": 26.075, + "completions/min_length": 19.4, + "completions/min_terminated_length": 19.4, + "epoch": 0.28888888888888886, "frac_reward_zero_std": 0.0, - "grad_norm": 1.149415373802185, - "kl": 0.00938094645498495, - "learning_rate": 3.761904761904762e-06, - "loss": 0.0003751114010810852, - "num_tokens": 266985.0, - "reward": 0.5003499746322632, - "reward_std": 0.16892780363559723, - "rewards/reward_fn/mean": 0.5003500044345855, - "rewards/reward_fn/std": 0.14663992822170258, + "grad_norm": 1.2825119495391846, + "kl": 0.07132846353342756, + "learning_rate": 3.779342723004695e-06, + "loss": 0.0028530970215797425, + "num_tokens": 267279.0, + "reward": 0.5383000016212464, + "reward_std": 0.20746512711048126, + "rewards/reward_fn/mean": 0.5383000195026397, + "rewards/reward_fn/std": 0.1835445523262024, "step": 65 }, { @@ -366,25 +366,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 27.0, + "completion_length": 27.925, "completions/clipped_ratio": 0.0, - "completions/max_length": 40.6, - "completions/max_terminated_length": 40.6, - "completions/mean_length": 27.0, - "completions/mean_terminated_length": 27.0, - "completions/min_length": 17.8, - "completions/min_terminated_length": 17.8, - "epoch": 0.3153153153153153, + "completions/max_length": 44.0, + "completions/max_terminated_length": 44.0, + "completions/mean_length": 27.925, + "completions/mean_terminated_length": 27.925, + "completions/min_length": 22.0, + "completions/min_terminated_length": 22.0, + "epoch": 0.3111111111111111, "frac_reward_zero_std": 0.0, - "grad_norm": 0.9893133640289307, - "kl": 0.010053556466664304, - "learning_rate": 3.642857142857143e-06, - "loss": 0.00040204524993896483, - "num_tokens": 288099.0, - "reward": 0.4705499827861786, - "reward_std": 0.16256383657455445, - "rewards/reward_fn/mean": 0.470550000667572, - "rewards/reward_fn/std": 0.13094386458396912, + "grad_norm": 0.5832120776176453, + "kl": 0.05197672065114602, + "learning_rate": 3.6619718309859158e-06, + "loss": 0.0020790368318557738, + "num_tokens": 287984.0, + "reward": 0.5424999892711639, + "reward_std": 0.18002938330173493, + "rewards/reward_fn/mean": 0.5424999952316284, + "rewards/reward_fn/std": 0.18769851326942444, "step": 70 }, { @@ -393,25 +393,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 25.75, - "completions/clipped_ratio": 0.0, - "completions/max_length": 34.8, - "completions/max_terminated_length": 34.8, - "completions/mean_length": 25.75, - "completions/mean_terminated_length": 25.75, - "completions/min_length": 19.8, - "completions/min_terminated_length": 19.8, - "epoch": 0.33783783783783783, + "completion_length": 27.6, + "completions/clipped_ratio": 0.025, + "completions/max_length": 44.6, + "completions/max_terminated_length": 30.6, + "completions/mean_length": 27.6, + "completions/mean_terminated_length": 25.807143020629884, + "completions/min_length": 22.2, + "completions/min_terminated_length": 22.2, + "epoch": 0.3333333333333333, "frac_reward_zero_std": 0.0, - "grad_norm": 1.063340187072754, - "kl": 0.008550376921630232, - "learning_rate": 3.523809523809524e-06, - "loss": 0.00034204572439193725, - "num_tokens": 308531.0, - "reward": 0.5000500082969666, - "reward_std": 0.16242243051528932, - "rewards/reward_fn/mean": 0.5000500082969666, - "rewards/reward_fn/std": 0.14417539685964584, + "grad_norm": 0.6677760481834412, + "kl": 0.03671608620206825, + "learning_rate": 3.5446009389671364e-06, + "loss": 0.0014685407280921937, + "num_tokens": 307800.0, + "reward": 0.5715999841690064, + "reward_std": 0.18639334440231323, + "rewards/reward_fn/mean": 0.5715999960899353, + "rewards/reward_fn/std": 0.19429495334625244, "step": 75 }, { @@ -420,25 +420,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 27.025, + "completion_length": 25.6, "completions/clipped_ratio": 0.0, - "completions/max_length": 40.0, - "completions/max_terminated_length": 40.0, - "completions/mean_length": 27.025, - "completions/mean_terminated_length": 27.025, - "completions/min_length": 19.4, - "completions/min_terminated_length": 19.4, - "epoch": 0.36036036036036034, + "completions/max_length": 30.2, + "completions/max_terminated_length": 30.2, + "completions/mean_length": 25.6, + "completions/mean_terminated_length": 25.6, + "completions/min_length": 21.4, + "completions/min_terminated_length": 21.4, + "epoch": 0.35555555555555557, "frac_reward_zero_std": 0.0, - "grad_norm": 1.2858343124389648, - "kl": 0.019255010827328077, - "learning_rate": 3.404761904761905e-06, - "loss": 0.0007702425122261047, - "num_tokens": 328874.0, - "reward": 0.5261500000953674, - "reward_std": 0.11886464804410934, - "rewards/reward_fn/mean": 0.5261499881744385, - "rewards/reward_fn/std": 0.12691220343112947, + "grad_norm": 0.7261527180671692, + "kl": 0.05616153636947274, + "learning_rate": 3.427230046948357e-06, + "loss": 0.0022463813424110413, + "num_tokens": 327794.0, + "reward": 0.5464499950408935, + "reward_std": 0.18264567852020264, + "rewards/reward_fn/mean": 0.5464499950408935, + "rewards/reward_fn/std": 0.17480863779783248, "step": 80 }, { @@ -447,25 +447,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 25.475, + "completion_length": 24.075, "completions/clipped_ratio": 0.0, - "completions/max_length": 33.0, - "completions/max_terminated_length": 33.0, - "completions/mean_length": 25.475, - "completions/mean_terminated_length": 25.475, - "completions/min_length": 19.0, - "completions/min_terminated_length": 19.0, - "epoch": 0.38288288288288286, + "completions/max_length": 29.6, + "completions/max_terminated_length": 29.6, + "completions/mean_length": 24.075, + "completions/mean_terminated_length": 24.075, + "completions/min_length": 16.6, + "completions/min_terminated_length": 16.6, + "epoch": 0.37777777777777777, "frac_reward_zero_std": 0.0, - "grad_norm": 1.118030309677124, - "kl": 0.01090274843845691, - "learning_rate": 3.285714285714286e-06, - "loss": 0.00043606162071228025, - "num_tokens": 348373.0, - "reward": 0.49950000643730164, - "reward_std": 0.15655344128608703, - "rewards/reward_fn/mean": 0.49950000643730164, - "rewards/reward_fn/std": 0.14320258051156998, + "grad_norm": 1.1086981296539307, + "kl": 0.07743949705036357, + "learning_rate": 3.3098591549295777e-06, + "loss": 0.0030974715948104857, + "num_tokens": 348233.0, + "reward": 0.5536999821662902, + "reward_std": 0.18073648810386658, + "rewards/reward_fn/mean": 0.5536999821662902, + "rewards/reward_fn/std": 0.1820658951997757, "step": 85 }, { @@ -474,25 +474,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 27.9, - "completions/clipped_ratio": 0.05, - "completions/max_length": 55.4, - "completions/max_terminated_length": 31.6, - "completions/mean_length": 27.9, - "completions/mean_terminated_length": 24.28571472167969, - "completions/min_length": 18.0, - "completions/min_terminated_length": 18.0, - "epoch": 0.40540540540540543, + "completion_length": 25.375, + "completions/clipped_ratio": 0.0, + "completions/max_length": 34.4, + "completions/max_terminated_length": 34.4, + "completions/mean_length": 25.375, + "completions/mean_terminated_length": 25.375, + "completions/min_length": 18.6, + "completions/min_terminated_length": 18.6, + "epoch": 0.4, "frac_reward_zero_std": 0.0, - "grad_norm": 0.6538166403770447, - "kl": 0.008556566922561615, - "learning_rate": 3.1666666666666667e-06, - "loss": 0.0003422081470489502, - "num_tokens": 368815.0, - "reward": 0.5215500056743622, - "reward_std": 0.16581654250621797, - "rewards/reward_fn/mean": 0.5215500056743622, - "rewards/reward_fn/std": 0.15472394227981567, + "grad_norm": 1.1903071403503418, + "kl": 0.058934826811309904, + "learning_rate": 3.1924882629107983e-06, + "loss": 0.002357317507266998, + "num_tokens": 367732.0, + "reward": 0.5361499905586242, + "reward_std": 0.179675829410553, + "rewards/reward_fn/mean": 0.5361500144004822, + "rewards/reward_fn/std": 0.18574500381946563, "step": 90 }, { @@ -501,25 +501,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 25.75, + "completion_length": 26.825, "completions/clipped_ratio": 0.0, - "completions/max_length": 32.4, - "completions/max_terminated_length": 32.4, - "completions/mean_length": 25.75, - "completions/mean_terminated_length": 25.75, + "completions/max_length": 37.2, + "completions/max_terminated_length": 37.2, + "completions/mean_length": 26.825, + "completions/mean_terminated_length": 26.825, "completions/min_length": 20.8, "completions/min_terminated_length": 20.8, - "epoch": 0.42792792792792794, + "epoch": 0.4222222222222222, "frac_reward_zero_std": 0.0, - "grad_norm": 1.180251955986023, - "kl": 0.014757526386529208, - "learning_rate": 3.047619047619048e-06, - "loss": 0.0005902737379074096, - "num_tokens": 388761.0, - "reward": 0.514000016450882, - "reward_std": 0.1465125188231468, - "rewards/reward_fn/mean": 0.5139999985694885, - "rewards/reward_fn/std": 0.14701166301965712, + "grad_norm": 1.0952492952346802, + "kl": 0.040232469444163144, + "learning_rate": 3.075117370892019e-06, + "loss": 0.0016092658042907715, + "num_tokens": 387075.0, + "reward": 0.5278500020503998, + "reward_std": 0.1932522773742676, + "rewards/reward_fn/mean": 0.5278499901294709, + "rewards/reward_fn/std": 0.1796583503484726, "step": 95 }, { @@ -528,25 +528,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 26.35, - "completions/clipped_ratio": 0.025, - "completions/max_length": 46.6, - "completions/max_terminated_length": 32.4, - "completions/mean_length": 26.35, - "completions/mean_terminated_length": 24.517857360839844, - "completions/min_length": 17.8, - "completions/min_terminated_length": 17.8, - "epoch": 0.45045045045045046, + "completion_length": 26.125, + "completions/clipped_ratio": 0.0, + "completions/max_length": 35.6, + "completions/max_terminated_length": 35.6, + "completions/mean_length": 26.125, + "completions/mean_terminated_length": 26.125, + "completions/min_length": 22.6, + "completions/min_terminated_length": 22.6, + "epoch": 0.4444444444444444, "frac_reward_zero_std": 0.0, - "grad_norm": 0.8863919973373413, - "kl": 0.017311154678463937, - "learning_rate": 2.928571428571429e-06, - "loss": 0.0006924748420715332, - "num_tokens": 408309.0, - "reward": 0.5527999818325042, - "reward_std": 0.1593818634748459, - "rewards/reward_fn/mean": 0.5527999699115753, - "rewards/reward_fn/std": 0.16823574155569077, + "grad_norm": 0.8072858452796936, + "kl": 0.06188266044482589, + "learning_rate": 2.9577464788732396e-06, + "loss": 0.0024752289056777952, + "num_tokens": 406884.0, + "reward": 0.5849499881267548, + "reward_std": 0.20499025285243988, + "rewards/reward_fn/mean": 0.5849500000476837, + "rewards/reward_fn/std": 0.2056175708770752, "step": 100 }, { @@ -555,25 +555,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 28.675, - "completions/clipped_ratio": 0.05, - "completions/max_length": 45.6, - "completions/max_terminated_length": 33.2, - "completions/mean_length": 28.675, - "completions/mean_terminated_length": 25.291666793823243, - "completions/min_length": 20.6, - "completions/min_terminated_length": 20.6, - "epoch": 0.47297297297297297, + "completion_length": 26.45, + "completions/clipped_ratio": 0.025, + "completions/max_length": 43.2, + "completions/max_terminated_length": 29.0, + "completions/mean_length": 26.45, + "completions/mean_terminated_length": 24.639286041259766, + "completions/min_length": 20.4, + "completions/min_terminated_length": 20.4, + "epoch": 0.4666666666666667, "frac_reward_zero_std": 0.0, - "grad_norm": 0.7957925200462341, - "kl": 0.021478149453469087, - "learning_rate": 2.8095238095238096e-06, - "loss": 0.0008591204881668091, - "num_tokens": 429872.0, - "reward": 0.5294999718666077, - "reward_std": 0.15301789939403534, - "rewards/reward_fn/mean": 0.5294999837875366, - "rewards/reward_fn/std": 0.15534335970878602, + "grad_norm": 0.9750375747680664, + "kl": 0.0710214663646184, + "learning_rate": 2.8403755868544603e-06, + "loss": 0.0028407976031303407, + "num_tokens": 427518.0, + "reward": 0.5496499896049499, + "reward_std": 0.16298811435699462, + "rewards/reward_fn/mean": 0.5496499896049499, + "rewards/reward_fn/std": 0.15672676265239716, "step": 105 }, { @@ -582,25 +582,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 27.325, + "completion_length": 25.925, "completions/clipped_ratio": 0.0, - "completions/max_length": 47.2, - "completions/max_terminated_length": 47.2, - "completions/mean_length": 27.325, - "completions/mean_terminated_length": 27.325, - "completions/min_length": 19.2, - "completions/min_terminated_length": 19.2, - "epoch": 0.4954954954954955, + "completions/max_length": 31.8, + "completions/max_terminated_length": 31.8, + "completions/mean_length": 25.925, + "completions/mean_terminated_length": 25.925, + "completions/min_length": 21.6, + "completions/min_terminated_length": 21.6, + "epoch": 0.4888888888888889, "frac_reward_zero_std": 0.0, - "grad_norm": 1.187671184539795, - "kl": 0.01666262859798735, - "learning_rate": 2.6904761904761906e-06, - "loss": 0.0006665512919425964, - "num_tokens": 450229.0, - "reward": 0.5364000022411346, - "reward_std": 0.16447303295135499, - "rewards/reward_fn/mean": 0.5364000022411346, - "rewards/reward_fn/std": 0.17169796973466872, + "grad_norm": 1.0184260606765747, + "kl": 0.0396142341895029, + "learning_rate": 2.723004694835681e-06, + "loss": 0.0015845373272895813, + "num_tokens": 447807.0, + "reward": 0.5718499898910523, + "reward_std": 0.20725298821926116, + "rewards/reward_fn/mean": 0.5718500018119812, + "rewards/reward_fn/std": 0.214348441362381, "step": 110 }, { @@ -609,25 +609,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 27.6, - "completions/clipped_ratio": 0.025, - "completions/max_length": 46.6, - "completions/max_terminated_length": 37.2, - "completions/mean_length": 27.6, - "completions/mean_terminated_length": 25.853571701049805, - "completions/min_length": 18.8, - "completions/min_terminated_length": 18.8, - "epoch": 0.5180180180180181, + "completion_length": 26.225, + "completions/clipped_ratio": 0.0, + "completions/max_length": 31.2, + "completions/max_terminated_length": 31.2, + "completions/mean_length": 26.225, + "completions/mean_terminated_length": 26.225, + "completions/min_length": 22.4, + "completions/min_terminated_length": 22.4, + "epoch": 0.5111111111111111, "frac_reward_zero_std": 0.0, - "grad_norm": 1.3785444498062134, - "kl": 0.015608730388339608, - "learning_rate": 2.571428571428571e-06, - "loss": 0.0006243243813514709, - "num_tokens": 470911.0, - "reward": 0.5131499886512756, - "reward_std": 0.15916974246501922, - "rewards/reward_fn/mean": 0.513150018453598, - "rewards/reward_fn/std": 0.15986726433038712, + "grad_norm": 0.8158187866210938, + "kl": 0.048210305260727185, + "learning_rate": 2.6056338028169015e-06, + "loss": 0.0019283831119537354, + "num_tokens": 468650.0, + "reward": 0.5343500077724457, + "reward_std": 0.1919794887304306, + "rewards/reward_fn/mean": 0.5343500018119812, + "rewards/reward_fn/std": 0.17689327299594879, "step": 115 }, { @@ -636,25 +636,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 26.725, - "completions/clipped_ratio": 0.025, - "completions/max_length": 43.8, - "completions/max_terminated_length": 30.4, - "completions/mean_length": 26.725, - "completions/mean_terminated_length": 24.953571701049803, - "completions/min_length": 20.4, - "completions/min_terminated_length": 20.4, - "epoch": 0.5405405405405406, - "frac_reward_zero_std": 0.0, - "grad_norm": 0.8001337647438049, - "kl": 0.016426509176380933, - "learning_rate": 2.4523809523809526e-06, - "loss": 0.0006570681929588318, - "num_tokens": 490616.0, - "reward": 0.5314500093460083, - "reward_std": 0.16807928085327148, - "rewards/reward_fn/mean": 0.5314499974250794, - "rewards/reward_fn/std": 0.16840155124664308, + "completion_length": 25.875, + "completions/clipped_ratio": 0.0, + "completions/max_length": 30.0, + "completions/max_terminated_length": 30.0, + "completions/mean_length": 25.875, + "completions/mean_terminated_length": 25.875, + "completions/min_length": 24.2, + "completions/min_terminated_length": 24.2, + "epoch": 0.5333333333333333, + "frac_reward_zero_std": 0.05, + "grad_norm": 0.8932302594184875, + "kl": 0.05402324852766469, + "learning_rate": 2.488262910798122e-06, + "loss": 0.0021608427166938783, + "num_tokens": 489051.0, + "reward": 0.6121499896049499, + "reward_std": 0.16680648624897004, + "rewards/reward_fn/mean": 0.612150001525879, + "rewards/reward_fn/std": 0.18321824073791504, "step": 120 }, { @@ -663,25 +663,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 25.8, - "completions/clipped_ratio": 0.0, - "completions/max_length": 39.8, - "completions/max_terminated_length": 39.8, - "completions/mean_length": 25.8, - "completions/mean_terminated_length": 25.8, - "completions/min_length": 17.6, - "completions/min_terminated_length": 17.6, - "epoch": 0.5630630630630631, + "completion_length": 27.3, + "completions/clipped_ratio": 0.025, + "completions/max_length": 42.6, + "completions/max_terminated_length": 29.8, + "completions/mean_length": 27.3, + "completions/mean_terminated_length": 25.514286041259766, + "completions/min_length": 21.4, + "completions/min_terminated_length": 21.4, + "epoch": 0.5555555555555556, "frac_reward_zero_std": 0.0, - "grad_norm": 1.0382933616638184, - "kl": 0.016185989990481174, - "learning_rate": 2.3333333333333336e-06, - "loss": 0.000647495687007904, - "num_tokens": 509092.0, - "reward": 0.5144500017166138, - "reward_std": 0.15323003232479096, - "rewards/reward_fn/mean": 0.5144500017166138, - "rewards/reward_fn/std": 0.13980331867933274, + "grad_norm": 0.922471821308136, + "kl": 0.04857689954806119, + "learning_rate": 2.370892018779343e-06, + "loss": 0.0019433587789535523, + "num_tokens": 508231.0, + "reward": 0.5884499907493591, + "reward_std": 0.17119054943323136, + "rewards/reward_fn/mean": 0.588450014591217, + "rewards/reward_fn/std": 0.18389662504196166, "step": 125 }, { @@ -690,25 +690,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 26.275, + "completion_length": 26.05, "completions/clipped_ratio": 0.0, - "completions/max_length": 36.4, - "completions/max_terminated_length": 36.4, - "completions/mean_length": 26.275, - "completions/mean_terminated_length": 26.275, - "completions/min_length": 19.6, - "completions/min_terminated_length": 19.6, - "epoch": 0.5855855855855856, + "completions/max_length": 31.0, + "completions/max_terminated_length": 31.0, + "completions/mean_length": 26.05, + "completions/mean_terminated_length": 26.05, + "completions/min_length": 22.0, + "completions/min_terminated_length": 22.0, + "epoch": 0.5777777777777777, "frac_reward_zero_std": 0.0, - "grad_norm": 1.3675200939178467, - "kl": 0.01404028357937932, - "learning_rate": 2.2142857142857146e-06, - "loss": 0.0005615666508674621, - "num_tokens": 531083.0, - "reward": 0.5205999791622162, - "reward_std": 0.1824335426092148, - "rewards/reward_fn/mean": 0.520600003004074, - "rewards/reward_fn/std": 0.16652330607175828, + "grad_norm": 0.7222557067871094, + "kl": 0.06997429557377473, + "learning_rate": 2.2535211267605635e-06, + "loss": 0.0027989834547042848, + "num_tokens": 528729.0, + "reward": 0.5927999973297119, + "reward_std": 0.22429427206516267, + "rewards/reward_fn/mean": 0.5928000092506409, + "rewards/reward_fn/std": 0.21314262747764587, "step": 130 }, { @@ -717,25 +717,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 27.025, + "completion_length": 25.925, "completions/clipped_ratio": 0.0, - "completions/max_length": 41.0, - "completions/max_terminated_length": 41.0, - "completions/mean_length": 27.025, - "completions/mean_terminated_length": 27.025, - "completions/min_length": 20.2, - "completions/min_terminated_length": 20.2, - "epoch": 0.6081081081081081, + "completions/max_length": 35.6, + "completions/max_terminated_length": 35.6, + "completions/mean_length": 25.925, + "completions/mean_terminated_length": 25.925, + "completions/min_length": 18.8, + "completions/min_terminated_length": 18.8, + "epoch": 0.6, "frac_reward_zero_std": 0.0, - "grad_norm": 1.5435948371887207, - "kl": 0.01980702848522924, - "learning_rate": 2.0952380952380955e-06, - "loss": 0.0007923290133476258, - "num_tokens": 551052.0, - "reward": 0.5506500005722046, - "reward_std": 0.15252292901277542, - "rewards/reward_fn/mean": 0.5506499886512757, - "rewards/reward_fn/std": 0.16045962125062943, + "grad_norm": 1.2586678266525269, + "kl": 0.08584307442652062, + "learning_rate": 2.136150234741784e-06, + "loss": 0.0034336388111114503, + "num_tokens": 548748.0, + "reward": 0.5738499760627747, + "reward_std": 0.17571603059768676, + "rewards/reward_fn/mean": 0.5738499999046326, + "rewards/reward_fn/std": 0.19411510229110718, "step": 135 }, { @@ -744,25 +744,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 24.875, + "completion_length": 25.9, "completions/clipped_ratio": 0.0, - "completions/max_length": 30.0, - "completions/max_terminated_length": 30.0, - "completions/mean_length": 24.875, - "completions/mean_terminated_length": 24.875, - "completions/min_length": 19.8, - "completions/min_terminated_length": 19.8, - "epoch": 0.6306306306306306, + "completions/max_length": 29.4, + "completions/max_terminated_length": 29.4, + "completions/mean_length": 25.9, + "completions/mean_terminated_length": 25.9, + "completions/min_length": 23.4, + "completions/min_terminated_length": 23.4, + "epoch": 0.6222222222222222, "frac_reward_zero_std": 0.0, - "grad_norm": 1.0660195350646973, - "kl": 0.014228896767599508, - "learning_rate": 1.976190476190476e-06, - "loss": 0.0005690708756446838, - "num_tokens": 570993.0, - "reward": 0.5141499876976013, - "reward_std": 0.1573312520980835, - "rewards/reward_fn/mean": 0.5141499936580658, - "rewards/reward_fn/std": 0.14991891831159593, + "grad_norm": 1.0420219898223877, + "kl": 0.06210445412434638, + "learning_rate": 2.0187793427230047e-06, + "loss": 0.0024841248989105223, + "num_tokens": 569094.0, + "reward": 0.6095999956130982, + "reward_std": 0.2001112163066864, + "rewards/reward_fn/mean": 0.6095999956130982, + "rewards/reward_fn/std": 0.18984024077653885, "step": 140 }, { @@ -771,25 +771,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 24.775, - "completions/clipped_ratio": 0.0, - "completions/max_length": 35.4, - "completions/max_terminated_length": 35.4, - "completions/mean_length": 24.775, - "completions/mean_terminated_length": 24.775, - "completions/min_length": 17.0, - "completions/min_terminated_length": 17.0, - "epoch": 0.6531531531531531, + "completion_length": 27.85, + "completions/clipped_ratio": 0.025, + "completions/max_length": 46.0, + "completions/max_terminated_length": 34.8, + "completions/mean_length": 27.85, + "completions/mean_terminated_length": 26.164286041259764, + "completions/min_length": 21.2, + "completions/min_terminated_length": 21.2, + "epoch": 0.6444444444444445, "frac_reward_zero_std": 0.0, - "grad_norm": 0.9134359359741211, - "kl": 0.015506639698287472, - "learning_rate": 1.8571428571428573e-06, - "loss": 0.0006200879812240601, - "num_tokens": 591118.0, - "reward": 0.5515999913215637, - "reward_std": 0.1493409514427185, - "rewards/reward_fn/mean": 0.5516000032424927, - "rewards/reward_fn/std": 0.15984065383672713, + "grad_norm": 0.9585386514663696, + "kl": 0.04591481959214434, + "learning_rate": 1.9014084507042254e-06, + "loss": 0.0018365621566772462, + "num_tokens": 589932.0, + "reward": 0.5361499905586242, + "reward_std": 0.20909147560596467, + "rewards/reward_fn/mean": 0.5361500024795532, + "rewards/reward_fn/std": 0.1902428910136223, "step": 145 }, { @@ -798,25 +798,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 25.45, + "completion_length": 25.25, "completions/clipped_ratio": 0.0, - "completions/max_length": 34.6, - "completions/max_terminated_length": 34.6, - "completions/mean_length": 25.45, - "completions/mean_terminated_length": 25.45, - "completions/min_length": 20.6, - "completions/min_terminated_length": 20.6, - "epoch": 0.6756756756756757, + "completions/max_length": 29.6, + "completions/max_terminated_length": 29.6, + "completions/mean_length": 25.25, + "completions/mean_terminated_length": 25.25, + "completions/min_length": 21.6, + "completions/min_terminated_length": 21.6, + "epoch": 0.6666666666666666, "frac_reward_zero_std": 0.0, - "grad_norm": 1.011871099472046, - "kl": 0.018102088455634657, - "learning_rate": 1.738095238095238e-06, - "loss": 0.0007240131497383117, - "num_tokens": 611492.0, - "reward": 0.5036499917507171, - "reward_std": 0.14262343645095826, - "rewards/reward_fn/mean": 0.5036500096321106, - "rewards/reward_fn/std": 0.14135744124650956, + "grad_norm": 0.45393237471580505, + "kl": 0.04076897802297026, + "learning_rate": 1.784037558685446e-06, + "loss": 0.001630684733390808, + "num_tokens": 610754.0, + "reward": 0.5666499972343445, + "reward_std": 0.17826161682605743, + "rewards/reward_fn/mean": 0.5666500210762024, + "rewards/reward_fn/std": 0.1894826263189316, "step": 150 }, { @@ -825,25 +825,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 25.55, + "completion_length": 24.85, "completions/clipped_ratio": 0.0, - "completions/max_length": 35.4, - "completions/max_terminated_length": 35.4, - "completions/mean_length": 25.55, - "completions/mean_terminated_length": 25.55, - "completions/min_length": 19.2, - "completions/min_terminated_length": 19.2, - "epoch": 0.6981981981981982, + "completions/max_length": 29.2, + "completions/max_terminated_length": 29.2, + "completions/mean_length": 24.85, + "completions/mean_terminated_length": 24.85, + "completions/min_length": 21.4, + "completions/min_terminated_length": 21.4, + "epoch": 0.6888888888888889, "frac_reward_zero_std": 0.0, - "grad_norm": 1.136124849319458, - "kl": 0.02021147561608814, - "learning_rate": 1.6190476190476193e-06, - "loss": 0.0008084163069725037, - "num_tokens": 632232.0, - "reward": 0.5214999973773956, - "reward_std": 0.1552806466817856, - "rewards/reward_fn/mean": 0.5214999914169312, - "rewards/reward_fn/std": 0.14986062049865723, + "grad_norm": 1.101528525352478, + "kl": 0.0628763473010622, + "learning_rate": 1.6666666666666667e-06, + "loss": 0.0025150284171104433, + "num_tokens": 632434.0, + "reward": 0.5634499907493591, + "reward_std": 0.18957532048225403, + "rewards/reward_fn/mean": 0.5634500086307526, + "rewards/reward_fn/std": 0.1851608410477638, "step": 155 }, { @@ -854,23 +854,23 @@ "clip_ratio/region_mean": 0.0, "completion_length": 25.6, "completions/clipped_ratio": 0.0, - "completions/max_length": 34.2, - "completions/max_terminated_length": 34.2, + "completions/max_length": 30.6, + "completions/max_terminated_length": 30.6, "completions/mean_length": 25.6, "completions/mean_terminated_length": 25.6, - "completions/min_length": 19.6, - "completions/min_terminated_length": 19.6, - "epoch": 0.7207207207207207, + "completions/min_length": 22.4, + "completions/min_terminated_length": 22.4, + "epoch": 0.7111111111111111, "frac_reward_zero_std": 0.0, - "grad_norm": 0.7977616190910339, - "kl": 0.015592006998485886, - "learning_rate": 1.5e-06, - "loss": 0.0006236553192138672, - "num_tokens": 652692.0, - "reward": 0.5022999882698059, - "reward_std": 0.16786714345216752, - "rewards/reward_fn/mean": 0.5023000061511993, - "rewards/reward_fn/std": 0.1559548258781433, + "grad_norm": 0.593997061252594, + "kl": 0.08342576812719926, + "learning_rate": 1.5492957746478873e-06, + "loss": 0.0033370301127433775, + "num_tokens": 653686.0, + "reward": 0.5921499967575073, + "reward_std": 0.20258608162403108, + "rewards/reward_fn/mean": 0.5921500205993653, + "rewards/reward_fn/std": 0.20735768973827362, "step": 160 }, { @@ -879,25 +879,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 29.225, - "completions/clipped_ratio": 0.025, - "completions/max_length": 62.2, - "completions/max_terminated_length": 52.0, - "completions/mean_length": 29.225, - "completions/mean_terminated_length": 27.489286041259767, - "completions/min_length": 17.6, - "completions/min_terminated_length": 17.6, - "epoch": 0.7432432432432432, + "completion_length": 25.175, + "completions/clipped_ratio": 0.0, + "completions/max_length": 30.4, + "completions/max_terminated_length": 30.4, + "completions/mean_length": 25.175, + "completions/mean_terminated_length": 25.175, + "completions/min_length": 20.4, + "completions/min_terminated_length": 20.4, + "epoch": 0.7333333333333333, "frac_reward_zero_std": 0.0, - "grad_norm": 0.7920641899108887, - "kl": 0.02950997047009878, - "learning_rate": 1.3809523809523812e-06, - "loss": 0.0011803746223449708, - "num_tokens": 672595.0, - "reward": 0.5018999874591827, - "reward_std": 0.16178602278232573, - "rewards/reward_fn/mean": 0.5018999993801116, - "rewards/reward_fn/std": 0.1466786965727806, + "grad_norm": 0.8682815432548523, + "kl": 0.04647499453276396, + "learning_rate": 1.4319248826291082e-06, + "loss": 0.0018589019775390625, + "num_tokens": 673657.0, + "reward": 0.5476499795913696, + "reward_std": 0.18038293421268464, + "rewards/reward_fn/mean": 0.5476500034332276, + "rewards/reward_fn/std": 0.20103364586830139, "step": 165 }, { @@ -906,25 +906,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 25.225, + "completion_length": 24.95, "completions/clipped_ratio": 0.0, - "completions/max_length": 33.0, - "completions/max_terminated_length": 33.0, - "completions/mean_length": 25.225, - "completions/mean_terminated_length": 25.225, - "completions/min_length": 18.4, - "completions/min_terminated_length": 18.4, - "epoch": 0.7657657657657657, + "completions/max_length": 28.0, + "completions/max_terminated_length": 28.0, + "completions/mean_length": 24.95, + "completions/mean_terminated_length": 24.95, + "completions/min_length": 23.2, + "completions/min_terminated_length": 23.2, + "epoch": 0.7555555555555555, "frac_reward_zero_std": 0.0, - "grad_norm": 1.1562975645065308, - "kl": 0.018220309726893903, - "learning_rate": 1.261904761904762e-06, - "loss": 0.0007288455963134766, - "num_tokens": 692896.0, - "reward": 0.5815999805927277, - "reward_std": 0.14750247299671174, - "rewards/reward_fn/mean": 0.5815999805927277, - "rewards/reward_fn/std": 0.1825831338763237, + "grad_norm": 0.6287415027618408, + "kl": 0.0606904512271285, + "learning_rate": 1.3145539906103288e-06, + "loss": 0.0024275988340377807, + "num_tokens": 693125.0, + "reward": 0.6188999772071838, + "reward_std": 0.20350532233715057, + "rewards/reward_fn/mean": 0.6188999891281128, + "rewards/reward_fn/std": 0.2084890365600586, "step": 170 }, { @@ -933,25 +933,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 26.95, - "completions/clipped_ratio": 0.025, - "completions/max_length": 49.8, - "completions/max_terminated_length": 35.6, - "completions/mean_length": 26.95, - "completions/mean_terminated_length": 25.167857360839843, - "completions/min_length": 19.4, - "completions/min_terminated_length": 19.4, - "epoch": 0.7882882882882883, + "completion_length": 25.175, + "completions/clipped_ratio": 0.0, + "completions/max_length": 28.8, + "completions/max_terminated_length": 28.8, + "completions/mean_length": 25.175, + "completions/mean_terminated_length": 25.175, + "completions/min_length": 21.4, + "completions/min_terminated_length": 21.4, + "epoch": 0.7777777777777778, "frac_reward_zero_std": 0.0, - "grad_norm": 1.2884600162506104, - "kl": 0.021587877761339767, - "learning_rate": 1.142857142857143e-06, - "loss": 0.0008635029196739196, - "num_tokens": 712396.0, - "reward": 0.5006500005722045, - "reward_std": 0.17076628804206848, - "rewards/reward_fn/mean": 0.5006500005722045, - "rewards/reward_fn/std": 0.14878996163606645, + "grad_norm": 1.0185426473617554, + "kl": 0.044300994148943576, + "learning_rate": 1.1971830985915492e-06, + "loss": 0.001772068440914154, + "num_tokens": 713356.0, + "reward": 0.49915000796318054, + "reward_std": 0.17090770602226257, + "rewards/reward_fn/mean": 0.4991500020027161, + "rewards/reward_fn/std": 0.15009358823299407, "step": 175 }, { @@ -960,25 +960,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 26.35, - "completions/clipped_ratio": 0.025, - "completions/max_length": 46.0, - "completions/max_terminated_length": 35.2, - "completions/mean_length": 26.35, - "completions/mean_terminated_length": 24.57142868041992, - "completions/min_length": 15.4, - "completions/min_terminated_length": 15.4, - "epoch": 0.8108108108108109, + "completion_length": 25.825, + "completions/clipped_ratio": 0.0, + "completions/max_length": 30.0, + "completions/max_terminated_length": 30.0, + "completions/mean_length": 25.825, + "completions/mean_terminated_length": 25.825, + "completions/min_length": 22.2, + "completions/min_terminated_length": 22.2, + "epoch": 0.8, "frac_reward_zero_std": 0.0, - "grad_norm": 1.3578517436981201, - "kl": 0.025513717756257392, - "learning_rate": 1.023809523809524e-06, - "loss": 0.001020367443561554, - "num_tokens": 732648.0, - "reward": 0.5190999984741211, - "reward_std": 0.14453262090682983, - "rewards/reward_fn/mean": 0.51910001039505, - "rewards/reward_fn/std": 0.14755631536245345, + "grad_norm": 0.8517215251922607, + "kl": 0.07110593506367877, + "learning_rate": 1.07981220657277e-06, + "loss": 0.0028442263603210447, + "num_tokens": 733613.0, + "reward": 0.5364999890327453, + "reward_std": 0.17903943061828614, + "rewards/reward_fn/mean": 0.5364999890327453, + "rewards/reward_fn/std": 0.1849100574851036, "step": 180 }, { @@ -987,25 +987,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 26.875, - "completions/clipped_ratio": 0.025, - "completions/max_length": 44.8, - "completions/max_terminated_length": 30.8, - "completions/mean_length": 26.875, - "completions/mean_terminated_length": 25.082143020629882, - "completions/min_length": 20.2, - "completions/min_terminated_length": 20.2, - "epoch": 0.8333333333333334, + "completion_length": 25.725, + "completions/clipped_ratio": 0.0, + "completions/max_length": 33.2, + "completions/max_terminated_length": 33.2, + "completions/mean_length": 25.725, + "completions/mean_terminated_length": 25.725, + "completions/min_length": 21.8, + "completions/min_terminated_length": 21.8, + "epoch": 0.8222222222222222, "frac_reward_zero_std": 0.0, - "grad_norm": 1.042319893836975, - "kl": 0.014541034388821572, - "learning_rate": 9.047619047619048e-07, - "loss": 0.0005815878510475158, - "num_tokens": 752821.0, - "reward": 0.49784999489784243, - "reward_std": 0.16807928085327148, - "rewards/reward_fn/mean": 0.49785000681877134, - "rewards/reward_fn/std": 0.14966378808021547, + "grad_norm": 1.2276585102081299, + "kl": 0.06446516590658576, + "learning_rate": 9.624413145539907e-07, + "loss": 0.002578553557395935, + "num_tokens": 753214.0, + "reward": 0.5435999751091003, + "reward_std": 0.1755038946866989, + "rewards/reward_fn/mean": 0.5436000108718873, + "rewards/reward_fn/std": 0.1861019790172577, "step": 185 }, { @@ -1014,25 +1014,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 25.625, + "completion_length": 24.6, "completions/clipped_ratio": 0.0, - "completions/max_length": 35.6, - "completions/max_terminated_length": 35.6, - "completions/mean_length": 25.625, - "completions/mean_terminated_length": 25.625, - "completions/min_length": 19.4, - "completions/min_terminated_length": 19.4, - "epoch": 0.8558558558558559, + "completions/max_length": 27.4, + "completions/max_terminated_length": 27.4, + "completions/mean_length": 24.6, + "completions/mean_terminated_length": 24.6, + "completions/min_length": 20.8, + "completions/min_terminated_length": 20.8, + "epoch": 0.8444444444444444, "frac_reward_zero_std": 0.0, - "grad_norm": 0.8876456022262573, - "kl": 0.017240419032168573, - "learning_rate": 7.857142857142857e-07, - "loss": 0.0006895914673805236, - "num_tokens": 772888.0, - "reward": 0.4967499911785126, - "reward_std": 0.16355379521846772, - "rewards/reward_fn/mean": 0.4967499911785126, - "rewards/reward_fn/std": 0.14962645918130874, + "grad_norm": 0.9992023706436157, + "kl": 0.05620210377383046, + "learning_rate": 8.450704225352114e-07, + "loss": 0.0022480204701423646, + "num_tokens": 774816.0, + "reward": 0.5924999952316284, + "reward_std": 0.1981313169002533, + "rewards/reward_fn/mean": 0.5924999952316284, + "rewards/reward_fn/std": 0.19494172781705857, "step": 190 }, { @@ -1041,25 +1041,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 26.8, - "completions/clipped_ratio": 0.025, - "completions/max_length": 45.2, - "completions/max_terminated_length": 31.2, - "completions/mean_length": 26.8, - "completions/mean_terminated_length": 25.014286041259766, - "completions/min_length": 18.4, - "completions/min_terminated_length": 18.4, - "epoch": 0.8783783783783784, + "completion_length": 25.925, + "completions/clipped_ratio": 0.0, + "completions/max_length": 32.8, + "completions/max_terminated_length": 32.8, + "completions/mean_length": 25.925, + "completions/mean_terminated_length": 25.925, + "completions/min_length": 21.6, + "completions/min_terminated_length": 21.6, + "epoch": 0.8666666666666667, "frac_reward_zero_std": 0.0, - "grad_norm": 1.3262248039245605, - "kl": 0.01487510468577966, - "learning_rate": 6.666666666666667e-07, - "loss": 0.0005949988961219787, - "num_tokens": 792706.0, - "reward": 0.5399499893188476, - "reward_std": 0.17628171145915986, - "rewards/reward_fn/mean": 0.5399500131607056, - "rewards/reward_fn/std": 0.1639223352074623, + "grad_norm": 0.6905612349510193, + "kl": 0.061457820073701444, + "learning_rate": 7.27699530516432e-07, + "loss": 0.0024582624435424806, + "num_tokens": 794225.0, + "reward": 0.5802499890327454, + "reward_std": 0.17005917578935623, + "rewards/reward_fn/mean": 0.5802499890327454, + "rewards/reward_fn/std": 0.18120778799057008, "step": 195 }, { @@ -1068,31 +1068,31 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 25.4, + "completion_length": 25.975, "completions/clipped_ratio": 0.0, - "completions/max_length": 34.8, - "completions/max_terminated_length": 34.8, - "completions/mean_length": 25.4, - "completions/mean_terminated_length": 25.4, - "completions/min_length": 18.0, - "completions/min_terminated_length": 18.0, - "epoch": 0.9009009009009009, + "completions/max_length": 31.6, + "completions/max_terminated_length": 31.6, + "completions/mean_length": 25.975, + "completions/mean_terminated_length": 25.975, + "completions/min_length": 23.6, + "completions/min_terminated_length": 23.6, + "epoch": 0.8888888888888888, "frac_reward_zero_std": 0.0, - "grad_norm": 1.9707448482513428, - "kl": 0.02140549005125649, - "learning_rate": 5.476190476190477e-07, - "loss": 0.0008562013506889343, - "num_tokens": 813816.0, - "reward": 0.5184999942779541, - "reward_std": 0.15726054608821868, - "rewards/reward_fn/mean": 0.5184999942779541, - "rewards/reward_fn/std": 0.15672532767057418, + "grad_norm": 0.7173504829406738, + "kl": 0.06312856795266271, + "learning_rate": 6.103286384976526e-07, + "loss": 0.0025250956416130064, + "num_tokens": 814240.0, + "reward": 0.5612499833106994, + "reward_std": 0.20499025285243988, + "rewards/reward_fn/mean": 0.561249989271164, + "rewards/reward_fn/std": 0.2024638831615448, "step": 200 } ], "logging_steps": 5, - "max_steps": 222, - "num_input_tokens_seen": 813816, + "max_steps": 225, + "num_input_tokens_seen": 814240, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { diff --git a/results/phase1/checkpoint-225/README.md b/results/phase1/checkpoint-225/README.md new file mode 100644 index 0000000000000000000000000000000000000000..93a54877d7d12fb86dbb10111c6d07e97a31b3ea --- /dev/null +++ b/results/phase1/checkpoint-225/README.md @@ -0,0 +1,211 @@ +--- +base_model: unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit +- grpo +- lora +- sft +- transformers +- trl +- unsloth +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/results/phase1/checkpoint-225/adapter_config.json b/results/phase1/checkpoint-225/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..d3cb3463c7f8c805e8cd8c25e64f3af87d6a00f3 --- /dev/null +++ b/results/phase1/checkpoint-225/adapter_config.json @@ -0,0 +1,50 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": { + "base_model_class": "Qwen2ForCausalLM", + "parent_library": "transformers.models.qwen2.modeling_qwen2", + "unsloth_fixed": true + }, + "base_model_name_or_path": "unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "q_proj", + "k_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/results/phase1/checkpoint-225/adapter_model.safetensors b/results/phase1/checkpoint-225/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..09581caf47be4752636a0af973ec271be3c7854c --- /dev/null +++ b/results/phase1/checkpoint-225/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe1beb57ab89a2aeb9d572e64c9dedcaba34759e2e289c74a9a5464cb62a56c2 +size 73911112 diff --git a/results/phase1/checkpoint-225/chat_template.jinja b/results/phase1/checkpoint-225/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..bdf7919a96cfe43d50914a007b9c0877bd0ec27e --- /dev/null +++ b/results/phase1/checkpoint-225/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/results/phase1/checkpoint-225/optimizer.pt b/results/phase1/checkpoint-225/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..05f710adcfa796205f7ffa21a1b672fb28595ca8 --- /dev/null +++ b/results/phase1/checkpoint-225/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdbc4306df8ca94e7edeb1d150377cede486bf781c6dbed18072b6de15d3becd +size 37969669 diff --git a/results/phase1/checkpoint-225/rng_state.pth b/results/phase1/checkpoint-225/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..24a58d1f462d848e955fd961543806a6aaa24413 --- /dev/null +++ b/results/phase1/checkpoint-225/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:598d859fd8ba3e6ff2f91b82e9538778a790e49919047c7acc5dac0e0a10b77c +size 14645 diff --git a/results/phase1/checkpoint-225/scaler.pt b/results/phase1/checkpoint-225/scaler.pt new file mode 100644 index 0000000000000000000000000000000000000000..94f5cac012b4c84adcbbd79b576c9223b48de7c4 --- /dev/null +++ b/results/phase1/checkpoint-225/scaler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:702984c680199a0e28a46b8d953b4d6a6c82386d777f96e7c9bdcfa92e49f34f +size 1383 diff --git a/results/phase1/checkpoint-225/scheduler.pt b/results/phase1/checkpoint-225/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..2fdb45f5291ed2137765ac3d3dc02098592e4fc4 --- /dev/null +++ b/results/phase1/checkpoint-225/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2432e75157bae001c5086109cac2c381c97313a80d5b81c47178c2570034aa82 +size 1465 diff --git a/results/phase1/checkpoint-225/tokenizer.json b/results/phase1/checkpoint-225/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..9fa63fd2d23a79aac969e7c1b933b0f71d2928c6 --- /dev/null +++ b/results/phase1/checkpoint-225/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af5891a15588546db1ac7f2baf8fa94835a51a85c032c39793a55bb048b47446 +size 11422523 diff --git a/results/phase1/checkpoint-225/tokenizer_config.json b/results/phase1/checkpoint-225/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..d3dac539765625f7e736bb2cdacac98bfa616b1f --- /dev/null +++ b/results/phase1/checkpoint-225/tokenizer_config.json @@ -0,0 +1,201 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "is_local": false, + "model_max_length": 32768, + "pad_token": "<|PAD_TOKEN|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151657": { + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151658": { + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151665": { + "content": "<|PAD_TOKEN|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + } + } +} diff --git a/results/phase1/checkpoint-225/trainer_state.json b/results/phase1/checkpoint-225/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..de3089fbc4dc79f43a53cca51bdcfc826fc7ea7d --- /dev/null +++ b/results/phase1/checkpoint-225/trainer_state.json @@ -0,0 +1,1249 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 225, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 25.075, + "completions/clipped_ratio": 0.0, + "completions/max_length": 30.8, + "completions/max_terminated_length": 30.8, + "completions/mean_length": 25.075, + "completions/mean_terminated_length": 25.075, + "completions/min_length": 19.4, + "completions/min_terminated_length": 19.4, + "epoch": 0.022222222222222223, + "frac_reward_zero_std": 0.6, + "grad_norm": 1.3092279434204102, + "kl": 0.03290070100920275, + "learning_rate": 1.6666666666666667e-06, + "loss": 0.0013160213828086853, + "num_tokens": 20275.0, + "reward": 0.48624999523162843, + "reward_std": 0.05833630859851837, + "rewards/reward_fn/mean": 0.4862500011920929, + "rewards/reward_fn/std": 0.04605271518230438, + "step": 5 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 25.925, + "completions/clipped_ratio": 0.0, + "completions/max_length": 33.0, + "completions/max_terminated_length": 33.0, + "completions/mean_length": 25.925, + "completions/mean_terminated_length": 25.925, + "completions/min_length": 19.4, + "completions/min_terminated_length": 19.4, + "epoch": 0.044444444444444446, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.4204450845718384, + "kl": 0.040759827199508436, + "learning_rate": 3.7500000000000005e-06, + "loss": 0.0016303554177284241, + "num_tokens": 41560.0, + "reward": 0.5283999860286712, + "reward_std": 0.17366542518138886, + "rewards/reward_fn/mean": 0.5283999919891358, + "rewards/reward_fn/std": 0.16809422075748442, + "step": 10 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 27.45, + "completions/clipped_ratio": 0.0, + "completions/max_length": 33.6, + "completions/max_terminated_length": 33.6, + "completions/mean_length": 27.45, + "completions/mean_terminated_length": 27.45, + "completions/min_length": 24.0, + "completions/min_terminated_length": 24.0, + "epoch": 0.06666666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9515441656112671, + "kl": 0.049849181214813146, + "learning_rate": 4.953051643192488e-06, + "loss": 0.0019938603043556215, + "num_tokens": 62246.0, + "reward": 0.4975499987602234, + "reward_std": 0.1564827263355255, + "rewards/reward_fn/mean": 0.49755001068115234, + "rewards/reward_fn/std": 0.14618260115385057, + "step": 15 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 26.525, + "completions/clipped_ratio": 0.0, + "completions/max_length": 33.0, + "completions/max_terminated_length": 33.0, + "completions/mean_length": 26.525, + "completions/mean_terminated_length": 26.525, + "completions/min_length": 22.2, + "completions/min_terminated_length": 22.2, + "epoch": 0.08888888888888889, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.6862999200820923, + "kl": 0.04350169296376407, + "learning_rate": 4.835680751173709e-06, + "loss": 0.0017400771379470826, + "num_tokens": 81479.0, + "reward": 0.5159000098705292, + "reward_std": 0.17465537190437316, + "rewards/reward_fn/mean": 0.5158999919891357, + "rewards/reward_fn/std": 0.16455023884773254, + "step": 20 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 28.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 40.6, + "completions/max_terminated_length": 40.6, + "completions/mean_length": 28.0, + "completions/mean_terminated_length": 28.0, + "completions/min_length": 22.6, + "completions/min_terminated_length": 22.6, + "epoch": 0.1111111111111111, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.7516958713531494, + "kl": 0.05824573401478119, + "learning_rate": 4.71830985915493e-06, + "loss": 0.002329717576503754, + "num_tokens": 102833.0, + "reward": 0.5547999978065491, + "reward_std": 0.18328206837177277, + "rewards/reward_fn/mean": 0.5547999858856201, + "rewards/reward_fn/std": 0.1974634051322937, + "step": 25 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 28.675, + "completions/clipped_ratio": 0.0, + "completions/max_length": 37.4, + "completions/max_terminated_length": 37.4, + "completions/mean_length": 28.675, + "completions/mean_terminated_length": 28.675, + "completions/min_length": 22.2, + "completions/min_terminated_length": 22.2, + "epoch": 0.13333333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.5611271858215332, + "kl": 0.09752920938190072, + "learning_rate": 4.60093896713615e-06, + "loss": 0.0039011374115943908, + "num_tokens": 123876.0, + "reward": 0.5165499806404114, + "reward_std": 0.16765501499176025, + "rewards/reward_fn/mean": 0.5165499806404114, + "rewards/reward_fn/std": 0.16748485416173936, + "step": 30 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 27.475, + "completions/clipped_ratio": 0.0, + "completions/max_length": 36.8, + "completions/max_terminated_length": 36.8, + "completions/mean_length": 27.475, + "completions/mean_terminated_length": 27.475, + "completions/min_length": 20.6, + "completions/min_terminated_length": 20.6, + "epoch": 0.15555555555555556, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9550014138221741, + "kl": 0.08032013729680329, + "learning_rate": 4.483568075117371e-06, + "loss": 0.003212757408618927, + "num_tokens": 145019.0, + "reward": 0.5004499793052674, + "reward_std": 0.18391846716403962, + "rewards/reward_fn/mean": 0.5004500031471253, + "rewards/reward_fn/std": 0.16948954164981841, + "step": 35 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 32.125, + "completions/clipped_ratio": 0.025, + "completions/max_length": 54.6, + "completions/max_terminated_length": 46.0, + "completions/mean_length": 32.125, + "completions/mean_terminated_length": 30.521428680419923, + "completions/min_length": 23.2, + "completions/min_terminated_length": 23.2, + "epoch": 0.17777777777777778, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1203869581222534, + "kl": 0.08563535290304572, + "learning_rate": 4.3661971830985915e-06, + "loss": 0.003425435721874237, + "num_tokens": 164800.0, + "reward": 0.5472500026226044, + "reward_std": 0.19579786211252212, + "rewards/reward_fn/mean": 0.5472500085830688, + "rewards/reward_fn/std": 0.19857290089130403, + "step": 40 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 27.875, + "completions/clipped_ratio": 0.0, + "completions/max_length": 35.0, + "completions/max_terminated_length": 35.0, + "completions/mean_length": 27.875, + "completions/mean_terminated_length": 27.875, + "completions/min_length": 21.4, + "completions/min_terminated_length": 21.4, + "epoch": 0.2, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.95042884349823, + "kl": 0.07295196709455923, + "learning_rate": 4.248826291079813e-06, + "loss": 0.0029180020093917845, + "num_tokens": 185761.0, + "reward": 0.5225999832153321, + "reward_std": 0.16489729881286622, + "rewards/reward_fn/mean": 0.5226000070571899, + "rewards/reward_fn/std": 0.16541497856378556, + "step": 45 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 27.875, + "completions/clipped_ratio": 0.0, + "completions/max_length": 35.2, + "completions/max_terminated_length": 35.2, + "completions/mean_length": 27.875, + "completions/mean_terminated_length": 27.875, + "completions/min_length": 21.4, + "completions/min_terminated_length": 21.4, + "epoch": 0.2222222222222222, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1037548780441284, + "kl": 0.08246680488809943, + "learning_rate": 4.131455399061034e-06, + "loss": 0.0032985761761665346, + "num_tokens": 206674.0, + "reward": 0.5245000004768372, + "reward_std": 0.1653215616941452, + "rewards/reward_fn/mean": 0.5245000004768372, + "rewards/reward_fn/std": 0.16877340227365495, + "step": 50 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 29.225, + "completions/clipped_ratio": 0.0, + "completions/max_length": 47.4, + "completions/max_terminated_length": 47.4, + "completions/mean_length": 29.225, + "completions/mean_terminated_length": 29.225, + "completions/min_length": 20.2, + "completions/min_terminated_length": 20.2, + "epoch": 0.24444444444444444, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8115681409835815, + "kl": 0.08297618771903217, + "learning_rate": 4.014084507042254e-06, + "loss": 0.0033189669251441956, + "num_tokens": 227351.0, + "reward": 0.5301500022411346, + "reward_std": 0.20612163245677947, + "rewards/reward_fn/mean": 0.5301500082015991, + "rewards/reward_fn/std": 0.19000594317913055, + "step": 55 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 27.475, + "completions/clipped_ratio": 0.025, + "completions/max_length": 45.0, + "completions/max_terminated_length": 31.8, + "completions/mean_length": 27.475, + "completions/mean_terminated_length": 25.646428680419923, + "completions/min_length": 18.8, + "completions/min_terminated_length": 18.8, + "epoch": 0.26666666666666666, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1722970008850098, + "kl": 0.06283304298995063, + "learning_rate": 3.896713615023475e-06, + "loss": 0.002513286471366882, + "num_tokens": 246464.0, + "reward": 0.513349997997284, + "reward_std": 0.18999958634376526, + "rewards/reward_fn/mean": 0.5133499920368194, + "rewards/reward_fn/std": 0.1748345360159874, + "step": 60 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 26.075, + "completions/clipped_ratio": 0.0, + "completions/max_length": 34.2, + "completions/max_terminated_length": 34.2, + "completions/mean_length": 26.075, + "completions/mean_terminated_length": 26.075, + "completions/min_length": 19.4, + "completions/min_terminated_length": 19.4, + "epoch": 0.28888888888888886, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2825119495391846, + "kl": 0.07132846353342756, + "learning_rate": 3.779342723004695e-06, + "loss": 0.0028530970215797425, + "num_tokens": 267279.0, + "reward": 0.5383000016212464, + "reward_std": 0.20746512711048126, + "rewards/reward_fn/mean": 0.5383000195026397, + "rewards/reward_fn/std": 0.1835445523262024, + "step": 65 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 27.925, + "completions/clipped_ratio": 0.0, + "completions/max_length": 44.0, + "completions/max_terminated_length": 44.0, + "completions/mean_length": 27.925, + "completions/mean_terminated_length": 27.925, + "completions/min_length": 22.0, + "completions/min_terminated_length": 22.0, + "epoch": 0.3111111111111111, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.5832120776176453, + "kl": 0.05197672065114602, + "learning_rate": 3.6619718309859158e-06, + "loss": 0.0020790368318557738, + "num_tokens": 287984.0, + "reward": 0.5424999892711639, + "reward_std": 0.18002938330173493, + "rewards/reward_fn/mean": 0.5424999952316284, + "rewards/reward_fn/std": 0.18769851326942444, + "step": 70 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 27.6, + "completions/clipped_ratio": 0.025, + "completions/max_length": 44.6, + "completions/max_terminated_length": 30.6, + "completions/mean_length": 27.6, + "completions/mean_terminated_length": 25.807143020629884, + "completions/min_length": 22.2, + "completions/min_terminated_length": 22.2, + "epoch": 0.3333333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.6677760481834412, + "kl": 0.03671608620206825, + "learning_rate": 3.5446009389671364e-06, + "loss": 0.0014685407280921937, + "num_tokens": 307800.0, + "reward": 0.5715999841690064, + "reward_std": 0.18639334440231323, + "rewards/reward_fn/mean": 0.5715999960899353, + "rewards/reward_fn/std": 0.19429495334625244, + "step": 75 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 25.6, + "completions/clipped_ratio": 0.0, + "completions/max_length": 30.2, + "completions/max_terminated_length": 30.2, + "completions/mean_length": 25.6, + "completions/mean_terminated_length": 25.6, + "completions/min_length": 21.4, + "completions/min_terminated_length": 21.4, + "epoch": 0.35555555555555557, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7261527180671692, + "kl": 0.05616153636947274, + "learning_rate": 3.427230046948357e-06, + "loss": 0.0022463813424110413, + "num_tokens": 327794.0, + "reward": 0.5464499950408935, + "reward_std": 0.18264567852020264, + "rewards/reward_fn/mean": 0.5464499950408935, + "rewards/reward_fn/std": 0.17480863779783248, + "step": 80 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 24.075, + "completions/clipped_ratio": 0.0, + "completions/max_length": 29.6, + "completions/max_terminated_length": 29.6, + "completions/mean_length": 24.075, + "completions/mean_terminated_length": 24.075, + "completions/min_length": 16.6, + "completions/min_terminated_length": 16.6, + "epoch": 0.37777777777777777, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1086981296539307, + "kl": 0.07743949705036357, + "learning_rate": 3.3098591549295777e-06, + "loss": 0.0030974715948104857, + "num_tokens": 348233.0, + "reward": 0.5536999821662902, + "reward_std": 0.18073648810386658, + "rewards/reward_fn/mean": 0.5536999821662902, + "rewards/reward_fn/std": 0.1820658951997757, + "step": 85 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 25.375, + "completions/clipped_ratio": 0.0, + "completions/max_length": 34.4, + "completions/max_terminated_length": 34.4, + "completions/mean_length": 25.375, + "completions/mean_terminated_length": 25.375, + "completions/min_length": 18.6, + "completions/min_terminated_length": 18.6, + "epoch": 0.4, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.1903071403503418, + "kl": 0.058934826811309904, + "learning_rate": 3.1924882629107983e-06, + "loss": 0.002357317507266998, + "num_tokens": 367732.0, + "reward": 0.5361499905586242, + "reward_std": 0.179675829410553, + "rewards/reward_fn/mean": 0.5361500144004822, + "rewards/reward_fn/std": 0.18574500381946563, + "step": 90 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 26.825, + "completions/clipped_ratio": 0.0, + "completions/max_length": 37.2, + "completions/max_terminated_length": 37.2, + "completions/mean_length": 26.825, + "completions/mean_terminated_length": 26.825, + "completions/min_length": 20.8, + "completions/min_terminated_length": 20.8, + "epoch": 0.4222222222222222, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0952492952346802, + "kl": 0.040232469444163144, + "learning_rate": 3.075117370892019e-06, + "loss": 0.0016092658042907715, + "num_tokens": 387075.0, + "reward": 0.5278500020503998, + "reward_std": 0.1932522773742676, + "rewards/reward_fn/mean": 0.5278499901294709, + "rewards/reward_fn/std": 0.1796583503484726, + "step": 95 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 26.125, + "completions/clipped_ratio": 0.0, + "completions/max_length": 35.6, + "completions/max_terminated_length": 35.6, + "completions/mean_length": 26.125, + "completions/mean_terminated_length": 26.125, + "completions/min_length": 22.6, + "completions/min_terminated_length": 22.6, + "epoch": 0.4444444444444444, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8072858452796936, + "kl": 0.06188266044482589, + "learning_rate": 2.9577464788732396e-06, + "loss": 0.0024752289056777952, + "num_tokens": 406884.0, + "reward": 0.5849499881267548, + "reward_std": 0.20499025285243988, + "rewards/reward_fn/mean": 0.5849500000476837, + "rewards/reward_fn/std": 0.2056175708770752, + "step": 100 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 26.45, + "completions/clipped_ratio": 0.025, + "completions/max_length": 43.2, + "completions/max_terminated_length": 29.0, + "completions/mean_length": 26.45, + "completions/mean_terminated_length": 24.639286041259766, + "completions/min_length": 20.4, + "completions/min_terminated_length": 20.4, + "epoch": 0.4666666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9750375747680664, + "kl": 0.0710214663646184, + "learning_rate": 2.8403755868544603e-06, + "loss": 0.0028407976031303407, + "num_tokens": 427518.0, + "reward": 0.5496499896049499, + "reward_std": 0.16298811435699462, + "rewards/reward_fn/mean": 0.5496499896049499, + "rewards/reward_fn/std": 0.15672676265239716, + "step": 105 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 25.925, + "completions/clipped_ratio": 0.0, + "completions/max_length": 31.8, + "completions/max_terminated_length": 31.8, + "completions/mean_length": 25.925, + "completions/mean_terminated_length": 25.925, + "completions/min_length": 21.6, + "completions/min_terminated_length": 21.6, + "epoch": 0.4888888888888889, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0184260606765747, + "kl": 0.0396142341895029, + "learning_rate": 2.723004694835681e-06, + "loss": 0.0015845373272895813, + "num_tokens": 447807.0, + "reward": 0.5718499898910523, + "reward_std": 0.20725298821926116, + "rewards/reward_fn/mean": 0.5718500018119812, + "rewards/reward_fn/std": 0.214348441362381, + "step": 110 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 26.225, + "completions/clipped_ratio": 0.0, + "completions/max_length": 31.2, + "completions/max_terminated_length": 31.2, + "completions/mean_length": 26.225, + "completions/mean_terminated_length": 26.225, + "completions/min_length": 22.4, + "completions/min_terminated_length": 22.4, + "epoch": 0.5111111111111111, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8158187866210938, + "kl": 0.048210305260727185, + "learning_rate": 2.6056338028169015e-06, + "loss": 0.0019283831119537354, + "num_tokens": 468650.0, + "reward": 0.5343500077724457, + "reward_std": 0.1919794887304306, + "rewards/reward_fn/mean": 0.5343500018119812, + "rewards/reward_fn/std": 0.17689327299594879, + "step": 115 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 25.875, + "completions/clipped_ratio": 0.0, + "completions/max_length": 30.0, + "completions/max_terminated_length": 30.0, + "completions/mean_length": 25.875, + "completions/mean_terminated_length": 25.875, + "completions/min_length": 24.2, + "completions/min_terminated_length": 24.2, + "epoch": 0.5333333333333333, + "frac_reward_zero_std": 0.05, + "grad_norm": 0.8932302594184875, + "kl": 0.05402324852766469, + "learning_rate": 2.488262910798122e-06, + "loss": 0.0021608427166938783, + "num_tokens": 489051.0, + "reward": 0.6121499896049499, + "reward_std": 0.16680648624897004, + "rewards/reward_fn/mean": 0.612150001525879, + "rewards/reward_fn/std": 0.18321824073791504, + "step": 120 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 27.3, + "completions/clipped_ratio": 0.025, + "completions/max_length": 42.6, + "completions/max_terminated_length": 29.8, + "completions/mean_length": 27.3, + "completions/mean_terminated_length": 25.514286041259766, + "completions/min_length": 21.4, + "completions/min_terminated_length": 21.4, + "epoch": 0.5555555555555556, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.922471821308136, + "kl": 0.04857689954806119, + "learning_rate": 2.370892018779343e-06, + "loss": 0.0019433587789535523, + "num_tokens": 508231.0, + "reward": 0.5884499907493591, + "reward_std": 0.17119054943323136, + "rewards/reward_fn/mean": 0.588450014591217, + "rewards/reward_fn/std": 0.18389662504196166, + "step": 125 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 26.05, + "completions/clipped_ratio": 0.0, + "completions/max_length": 31.0, + "completions/max_terminated_length": 31.0, + "completions/mean_length": 26.05, + "completions/mean_terminated_length": 26.05, + "completions/min_length": 22.0, + "completions/min_terminated_length": 22.0, + "epoch": 0.5777777777777777, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7222557067871094, + "kl": 0.06997429557377473, + "learning_rate": 2.2535211267605635e-06, + "loss": 0.0027989834547042848, + "num_tokens": 528729.0, + "reward": 0.5927999973297119, + "reward_std": 0.22429427206516267, + "rewards/reward_fn/mean": 0.5928000092506409, + "rewards/reward_fn/std": 0.21314262747764587, + "step": 130 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 25.925, + "completions/clipped_ratio": 0.0, + "completions/max_length": 35.6, + "completions/max_terminated_length": 35.6, + "completions/mean_length": 25.925, + "completions/mean_terminated_length": 25.925, + "completions/min_length": 18.8, + "completions/min_terminated_length": 18.8, + "epoch": 0.6, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2586678266525269, + "kl": 0.08584307442652062, + "learning_rate": 2.136150234741784e-06, + "loss": 0.0034336388111114503, + "num_tokens": 548748.0, + "reward": 0.5738499760627747, + "reward_std": 0.17571603059768676, + "rewards/reward_fn/mean": 0.5738499999046326, + "rewards/reward_fn/std": 0.19411510229110718, + "step": 135 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 25.9, + "completions/clipped_ratio": 0.0, + "completions/max_length": 29.4, + "completions/max_terminated_length": 29.4, + "completions/mean_length": 25.9, + "completions/mean_terminated_length": 25.9, + "completions/min_length": 23.4, + "completions/min_terminated_length": 23.4, + "epoch": 0.6222222222222222, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0420219898223877, + "kl": 0.06210445412434638, + "learning_rate": 2.0187793427230047e-06, + "loss": 0.0024841248989105223, + "num_tokens": 569094.0, + "reward": 0.6095999956130982, + "reward_std": 0.2001112163066864, + "rewards/reward_fn/mean": 0.6095999956130982, + "rewards/reward_fn/std": 0.18984024077653885, + "step": 140 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 27.85, + "completions/clipped_ratio": 0.025, + "completions/max_length": 46.0, + "completions/max_terminated_length": 34.8, + "completions/mean_length": 27.85, + "completions/mean_terminated_length": 26.164286041259764, + "completions/min_length": 21.2, + "completions/min_terminated_length": 21.2, + "epoch": 0.6444444444444445, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9585386514663696, + "kl": 0.04591481959214434, + "learning_rate": 1.9014084507042254e-06, + "loss": 0.0018365621566772462, + "num_tokens": 589932.0, + "reward": 0.5361499905586242, + "reward_std": 0.20909147560596467, + "rewards/reward_fn/mean": 0.5361500024795532, + "rewards/reward_fn/std": 0.1902428910136223, + "step": 145 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 25.25, + "completions/clipped_ratio": 0.0, + "completions/max_length": 29.6, + "completions/max_terminated_length": 29.6, + "completions/mean_length": 25.25, + "completions/mean_terminated_length": 25.25, + "completions/min_length": 21.6, + "completions/min_terminated_length": 21.6, + "epoch": 0.6666666666666666, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.45393237471580505, + "kl": 0.04076897802297026, + "learning_rate": 1.784037558685446e-06, + "loss": 0.001630684733390808, + "num_tokens": 610754.0, + "reward": 0.5666499972343445, + "reward_std": 0.17826161682605743, + "rewards/reward_fn/mean": 0.5666500210762024, + "rewards/reward_fn/std": 0.1894826263189316, + "step": 150 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 24.85, + "completions/clipped_ratio": 0.0, + "completions/max_length": 29.2, + "completions/max_terminated_length": 29.2, + "completions/mean_length": 24.85, + "completions/mean_terminated_length": 24.85, + "completions/min_length": 21.4, + "completions/min_terminated_length": 21.4, + "epoch": 0.6888888888888889, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.101528525352478, + "kl": 0.0628763473010622, + "learning_rate": 1.6666666666666667e-06, + "loss": 0.0025150284171104433, + "num_tokens": 632434.0, + "reward": 0.5634499907493591, + "reward_std": 0.18957532048225403, + "rewards/reward_fn/mean": 0.5634500086307526, + "rewards/reward_fn/std": 0.1851608410477638, + "step": 155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 25.6, + "completions/clipped_ratio": 0.0, + "completions/max_length": 30.6, + "completions/max_terminated_length": 30.6, + "completions/mean_length": 25.6, + "completions/mean_terminated_length": 25.6, + "completions/min_length": 22.4, + "completions/min_terminated_length": 22.4, + "epoch": 0.7111111111111111, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.593997061252594, + "kl": 0.08342576812719926, + "learning_rate": 1.5492957746478873e-06, + "loss": 0.0033370301127433775, + "num_tokens": 653686.0, + "reward": 0.5921499967575073, + "reward_std": 0.20258608162403108, + "rewards/reward_fn/mean": 0.5921500205993653, + "rewards/reward_fn/std": 0.20735768973827362, + "step": 160 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 25.175, + "completions/clipped_ratio": 0.0, + "completions/max_length": 30.4, + "completions/max_terminated_length": 30.4, + "completions/mean_length": 25.175, + "completions/mean_terminated_length": 25.175, + "completions/min_length": 20.4, + "completions/min_terminated_length": 20.4, + "epoch": 0.7333333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8682815432548523, + "kl": 0.04647499453276396, + "learning_rate": 1.4319248826291082e-06, + "loss": 0.0018589019775390625, + "num_tokens": 673657.0, + "reward": 0.5476499795913696, + "reward_std": 0.18038293421268464, + "rewards/reward_fn/mean": 0.5476500034332276, + "rewards/reward_fn/std": 0.20103364586830139, + "step": 165 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 24.95, + "completions/clipped_ratio": 0.0, + "completions/max_length": 28.0, + "completions/max_terminated_length": 28.0, + "completions/mean_length": 24.95, + "completions/mean_terminated_length": 24.95, + "completions/min_length": 23.2, + "completions/min_terminated_length": 23.2, + "epoch": 0.7555555555555555, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.6287415027618408, + "kl": 0.0606904512271285, + "learning_rate": 1.3145539906103288e-06, + "loss": 0.0024275988340377807, + "num_tokens": 693125.0, + "reward": 0.6188999772071838, + "reward_std": 0.20350532233715057, + "rewards/reward_fn/mean": 0.6188999891281128, + "rewards/reward_fn/std": 0.2084890365600586, + "step": 170 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 25.175, + "completions/clipped_ratio": 0.0, + "completions/max_length": 28.8, + "completions/max_terminated_length": 28.8, + "completions/mean_length": 25.175, + "completions/mean_terminated_length": 25.175, + "completions/min_length": 21.4, + "completions/min_terminated_length": 21.4, + "epoch": 0.7777777777777778, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.0185426473617554, + "kl": 0.044300994148943576, + "learning_rate": 1.1971830985915492e-06, + "loss": 0.001772068440914154, + "num_tokens": 713356.0, + "reward": 0.49915000796318054, + "reward_std": 0.17090770602226257, + "rewards/reward_fn/mean": 0.4991500020027161, + "rewards/reward_fn/std": 0.15009358823299407, + "step": 175 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 25.825, + "completions/clipped_ratio": 0.0, + "completions/max_length": 30.0, + "completions/max_terminated_length": 30.0, + "completions/mean_length": 25.825, + "completions/mean_terminated_length": 25.825, + "completions/min_length": 22.2, + "completions/min_terminated_length": 22.2, + "epoch": 0.8, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8517215251922607, + "kl": 0.07110593506367877, + "learning_rate": 1.07981220657277e-06, + "loss": 0.0028442263603210447, + "num_tokens": 733613.0, + "reward": 0.5364999890327453, + "reward_std": 0.17903943061828614, + "rewards/reward_fn/mean": 0.5364999890327453, + "rewards/reward_fn/std": 0.1849100574851036, + "step": 180 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 25.725, + "completions/clipped_ratio": 0.0, + "completions/max_length": 33.2, + "completions/max_terminated_length": 33.2, + "completions/mean_length": 25.725, + "completions/mean_terminated_length": 25.725, + "completions/min_length": 21.8, + "completions/min_terminated_length": 21.8, + "epoch": 0.8222222222222222, + "frac_reward_zero_std": 0.0, + "grad_norm": 1.2276585102081299, + "kl": 0.06446516590658576, + "learning_rate": 9.624413145539907e-07, + "loss": 0.002578553557395935, + "num_tokens": 753214.0, + "reward": 0.5435999751091003, + "reward_std": 0.1755038946866989, + "rewards/reward_fn/mean": 0.5436000108718873, + "rewards/reward_fn/std": 0.1861019790172577, + "step": 185 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 24.6, + "completions/clipped_ratio": 0.0, + "completions/max_length": 27.4, + "completions/max_terminated_length": 27.4, + "completions/mean_length": 24.6, + "completions/mean_terminated_length": 24.6, + "completions/min_length": 20.8, + "completions/min_terminated_length": 20.8, + "epoch": 0.8444444444444444, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9992023706436157, + "kl": 0.05620210377383046, + "learning_rate": 8.450704225352114e-07, + "loss": 0.0022480204701423646, + "num_tokens": 774816.0, + "reward": 0.5924999952316284, + "reward_std": 0.1981313169002533, + "rewards/reward_fn/mean": 0.5924999952316284, + "rewards/reward_fn/std": 0.19494172781705857, + "step": 190 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 25.925, + "completions/clipped_ratio": 0.0, + "completions/max_length": 32.8, + "completions/max_terminated_length": 32.8, + "completions/mean_length": 25.925, + "completions/mean_terminated_length": 25.925, + "completions/min_length": 21.6, + "completions/min_terminated_length": 21.6, + "epoch": 0.8666666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.6905612349510193, + "kl": 0.061457820073701444, + "learning_rate": 7.27699530516432e-07, + "loss": 0.0024582624435424806, + "num_tokens": 794225.0, + "reward": 0.5802499890327454, + "reward_std": 0.17005917578935623, + "rewards/reward_fn/mean": 0.5802499890327454, + "rewards/reward_fn/std": 0.18120778799057008, + "step": 195 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 25.975, + "completions/clipped_ratio": 0.0, + "completions/max_length": 31.6, + "completions/max_terminated_length": 31.6, + "completions/mean_length": 25.975, + "completions/mean_terminated_length": 25.975, + "completions/min_length": 23.6, + "completions/min_terminated_length": 23.6, + "epoch": 0.8888888888888888, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7173504829406738, + "kl": 0.06312856795266271, + "learning_rate": 6.103286384976526e-07, + "loss": 0.0025250956416130064, + "num_tokens": 814240.0, + "reward": 0.5612499833106994, + "reward_std": 0.20499025285243988, + "rewards/reward_fn/mean": 0.561249989271164, + "rewards/reward_fn/std": 0.2024638831615448, + "step": 200 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 24.75, + "completions/clipped_ratio": 0.0, + "completions/max_length": 28.8, + "completions/max_terminated_length": 28.8, + "completions/mean_length": 24.75, + "completions/mean_terminated_length": 24.75, + "completions/min_length": 20.8, + "completions/min_terminated_length": 20.8, + "epoch": 0.9111111111111111, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.7710515260696411, + "kl": 0.07052509421482682, + "learning_rate": 4.929577464788733e-07, + "loss": 0.0028209388256073, + "num_tokens": 834334.0, + "reward": 0.5948500037193298, + "reward_std": 0.17953440248966218, + "rewards/reward_fn/mean": 0.5948499917984009, + "rewards/reward_fn/std": 0.19131502211093904, + "step": 205 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 25.8, + "completions/clipped_ratio": 0.0, + "completions/max_length": 30.6, + "completions/max_terminated_length": 30.6, + "completions/mean_length": 25.8, + "completions/mean_terminated_length": 25.8, + "completions/min_length": 23.4, + "completions/min_terminated_length": 23.4, + "epoch": 0.9333333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8554002046585083, + "kl": 0.05014841896481812, + "learning_rate": 3.755868544600939e-07, + "loss": 0.0020059287548065186, + "num_tokens": 854900.0, + "reward": 0.5546000003814697, + "reward_std": 0.21057639718055726, + "rewards/reward_fn/mean": 0.5545999944210053, + "rewards/reward_fn/std": 0.20256412923336028, + "step": 210 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 25.475, + "completions/clipped_ratio": 0.0, + "completions/max_length": 29.4, + "completions/max_terminated_length": 29.4, + "completions/mean_length": 25.475, + "completions/mean_terminated_length": 25.475, + "completions/min_length": 23.2, + "completions/min_terminated_length": 23.2, + "epoch": 0.9555555555555556, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.8410727381706238, + "kl": 0.06370279549155385, + "learning_rate": 2.582159624413146e-07, + "loss": 0.0025480970740318297, + "num_tokens": 874953.0, + "reward": 0.5734000027179718, + "reward_std": 0.2160918265581131, + "rewards/reward_fn/mean": 0.5733999907970428, + "rewards/reward_fn/std": 0.20869273394346238, + "step": 215 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 25.9, + "completions/clipped_ratio": 0.0, + "completions/max_length": 34.8, + "completions/max_terminated_length": 34.8, + "completions/mean_length": 25.9, + "completions/mean_terminated_length": 25.9, + "completions/min_length": 20.8, + "completions/min_terminated_length": 20.8, + "epoch": 0.9777777777777777, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.9725803136825562, + "kl": 0.075754539296031, + "learning_rate": 1.4084507042253522e-07, + "loss": 0.0030301779508590697, + "num_tokens": 894699.0, + "reward": 0.5753999829292298, + "reward_std": 0.20675801634788513, + "rewards/reward_fn/mean": 0.5753999948501587, + "rewards/reward_fn/std": 0.2071171909570694, + "step": 220 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completion_length": 25.4, + "completions/clipped_ratio": 0.0, + "completions/max_length": 30.2, + "completions/max_terminated_length": 30.2, + "completions/mean_length": 25.4, + "completions/mean_terminated_length": 25.4, + "completions/min_length": 21.8, + "completions/min_terminated_length": 21.8, + "epoch": 1.0, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.6214760541915894, + "kl": 0.0598387235251721, + "learning_rate": 2.347417840375587e-08, + "loss": 0.0023935258388519285, + "num_tokens": 914809.0, + "reward": 0.599399995803833, + "reward_std": 0.20831365883350372, + "rewards/reward_fn/mean": 0.599399995803833, + "rewards/reward_fn/std": 0.20224641859531403, + "step": 225 + } + ], + "logging_steps": 5, + "max_steps": 225, + "num_input_tokens_seen": 914809, + "num_train_epochs": 1, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/results/phase1/checkpoint-225/training_args.bin b/results/phase1/checkpoint-225/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..3f2d958667f4a7713fb067194a368a599c3c7e1c --- /dev/null +++ b/results/phase1/checkpoint-225/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b53420ca071212a25df4a51528d52e34338221d4319e6e32ff795209b21413d +size 6673 diff --git a/results/phase1/checkpoint-50/README.md b/results/phase1/checkpoint-50/README.md index 002e352e3f4655c9f00611b74eefa7b221963ec8..93a54877d7d12fb86dbb10111c6d07e97a31b3ea 100644 --- a/results/phase1/checkpoint-50/README.md +++ b/results/phase1/checkpoint-50/README.md @@ -6,6 +6,7 @@ tags: - base_model:adapter:unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit - grpo - lora +- sft - transformers - trl - unsloth diff --git a/results/phase1/checkpoint-50/adapter_config.json b/results/phase1/checkpoint-50/adapter_config.json index 5e64881c1f2a3150228a608fb85ae9d9b6278fb9..d3cb3463c7f8c805e8cd8c25e64f3af87d6a00f3 100644 --- a/results/phase1/checkpoint-50/adapter_config.json +++ b/results/phase1/checkpoint-50/adapter_config.json @@ -33,13 +33,13 @@ "rank_pattern": {}, "revision": null, "target_modules": [ + "gate_proj", + "q_proj", "k_proj", - "up_proj", + "v_proj", "down_proj", "o_proj", - "v_proj", - "gate_proj", - "q_proj" + "up_proj" ], "target_parameters": null, "task_type": "CAUSAL_LM", diff --git a/results/phase1/checkpoint-50/adapter_model.safetensors b/results/phase1/checkpoint-50/adapter_model.safetensors index 42f124fcfdbb51264091bdf150f2f485d7305c74..2b304848297398176cbfc2dafee3c09e5640976d 100644 --- a/results/phase1/checkpoint-50/adapter_model.safetensors +++ b/results/phase1/checkpoint-50/adapter_model.safetensors @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0adbf9d63bca21b84d7699d12cc495ff6d76158cfc43146af81ec0b5b93d22e8 +oid sha256:810a279575518f5aa4d0cab9e75ceddaa94cf06e33533008d364a47d79267e69 size 73911112 diff --git a/results/phase1/checkpoint-50/optimizer.pt b/results/phase1/checkpoint-50/optimizer.pt index 66e2738b465c45d24edba08c5c11ae51c574aab6..b04936f002b5d0c60924610e2f01fd68dad912ef 100644 --- a/results/phase1/checkpoint-50/optimizer.pt +++ b/results/phase1/checkpoint-50/optimizer.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ed241d586f6e676b800ea6c9ca0946e4b50e6c56b18b8f340ed3c8b51991a66d +oid sha256:9bbcbd4511a82c9c6bdb43bad61356c0af918216f63f91e10fa7ed7e0488ef08 size 37969669 diff --git a/results/phase1/checkpoint-50/rng_state.pth b/results/phase1/checkpoint-50/rng_state.pth index 501caa9ef028573148e559cd31ea2ac3faacf63d..95c9a46021a10efe36108f3cec788240a5cb654c 100644 --- a/results/phase1/checkpoint-50/rng_state.pth +++ b/results/phase1/checkpoint-50/rng_state.pth @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:cd7a6e62201dd7f462f420cdacde3f7d6295b8e73708203688b8af13c5d4f5c2 +oid sha256:06dea39c5c2b8f2bd1c6a68214429e67c2ef23cf860ac79996068119077e351f size 14645 diff --git a/results/phase1/checkpoint-50/scheduler.pt b/results/phase1/checkpoint-50/scheduler.pt index 222aefe5b0e63a578556929397f6efe61ccaac10..34f65e8d2777a82fb6726181e7a8fa22e30379c0 100644 --- a/results/phase1/checkpoint-50/scheduler.pt +++ b/results/phase1/checkpoint-50/scheduler.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:823af046a154dcc15b21be64915c4a2d28a7165ca91fe16175e2f7897dc126b8 +oid sha256:123e450a8ef68e6d25122f141b1c70f4e336373980eca07d10a278f228dc2c80 size 1465 diff --git a/results/phase1/checkpoint-50/trainer_state.json b/results/phase1/checkpoint-50/trainer_state.json index 5425914dc5756654afec63b2a0631e64c97f6960..ebfb14388b309fecb5d4c85cdfcfdbee2f167658 100644 --- a/results/phase1/checkpoint-50/trainer_state.json +++ b/results/phase1/checkpoint-50/trainer_state.json @@ -2,7 +2,7 @@ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, - "epoch": 0.22522522522522523, + "epoch": 0.2222222222222222, "eval_steps": 500, "global_step": 50, "is_hyper_param_search": false, @@ -15,25 +15,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 29.05, - "completions/clipped_ratio": 0.05, - "completions/max_length": 44.2, - "completions/max_terminated_length": 31.0, - "completions/mean_length": 29.05, - "completions/mean_terminated_length": 25.583333587646486, - "completions/min_length": 21.4, - "completions/min_terminated_length": 21.4, - "epoch": 0.02252252252252252, - "frac_reward_zero_std": 0.0, - "grad_norm": 1.2577366828918457, - "kl": 7.414049605358741e-06, + "completion_length": 25.075, + "completions/clipped_ratio": 0.0, + "completions/max_length": 30.8, + "completions/max_terminated_length": 30.8, + "completions/mean_length": 25.075, + "completions/mean_terminated_length": 25.075, + "completions/min_length": 19.4, + "completions/min_terminated_length": 19.4, + "epoch": 0.022222222222222223, + "frac_reward_zero_std": 0.6, + "grad_norm": 1.3092279434204102, + "kl": 0.03290070100920275, "learning_rate": 1.6666666666666667e-06, - "loss": 2.8312206268310547e-07, - "num_tokens": 21378.0, - "reward": 0.4944999933242798, - "reward_std": 0.15089658200740813, - "rewards/reward_fn/mean": 0.49449999928474425, - "rewards/reward_fn/std": 0.12999328523874282, + "loss": 0.0013160213828086853, + "num_tokens": 20275.0, + "reward": 0.48624999523162843, + "reward_std": 0.05833630859851837, + "rewards/reward_fn/mean": 0.4862500011920929, + "rewards/reward_fn/std": 0.04605271518230438, "step": 5 }, { @@ -42,25 +42,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 29.175, + "completion_length": 25.925, "completions/clipped_ratio": 0.0, - "completions/max_length": 41.8, - "completions/max_terminated_length": 41.8, - "completions/mean_length": 29.175, - "completions/mean_terminated_length": 29.175, - "completions/min_length": 19.2, - "completions/min_terminated_length": 19.2, - "epoch": 0.04504504504504504, + "completions/max_length": 33.0, + "completions/max_terminated_length": 33.0, + "completions/mean_length": 25.925, + "completions/mean_terminated_length": 25.925, + "completions/min_length": 19.4, + "completions/min_terminated_length": 19.4, + "epoch": 0.044444444444444446, "frac_reward_zero_std": 0.0, - "grad_norm": 1.148110032081604, - "kl": 2.58529256058182e-05, + "grad_norm": 1.4204450845718384, + "kl": 0.040759827199508436, "learning_rate": 3.7500000000000005e-06, - "loss": 9.343028068542481e-07, - "num_tokens": 42709.0, - "reward": 0.48589999675750734, - "reward_std": 0.14184561967849732, - "rewards/reward_fn/mean": 0.4859000027179718, - "rewards/reward_fn/std": 0.12539554834365846, + "loss": 0.0016303554177284241, + "num_tokens": 41560.0, + "reward": 0.5283999860286712, + "reward_std": 0.17366542518138886, + "rewards/reward_fn/mean": 0.5283999919891358, + "rewards/reward_fn/std": 0.16809422075748442, "step": 10 }, { @@ -69,25 +69,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 28.825, + "completion_length": 27.45, "completions/clipped_ratio": 0.0, - "completions/max_length": 44.2, - "completions/max_terminated_length": 44.2, - "completions/mean_length": 28.825, - "completions/mean_terminated_length": 28.825, - "completions/min_length": 21.2, - "completions/min_terminated_length": 21.2, - "epoch": 0.06756756756756757, + "completions/max_length": 33.6, + "completions/max_terminated_length": 33.6, + "completions/mean_length": 27.45, + "completions/mean_terminated_length": 27.45, + "completions/min_length": 24.0, + "completions/min_terminated_length": 24.0, + "epoch": 0.06666666666666667, "frac_reward_zero_std": 0.0, - "grad_norm": 1.1520825624465942, - "kl": 0.0001522944080875277, - "learning_rate": 4.952380952380953e-06, - "loss": 6.105005741119385e-06, - "num_tokens": 63008.0, - "reward": 0.4894999861717224, - "reward_std": 0.15768481492996217, - "rewards/reward_fn/mean": 0.48950000405311583, - "rewards/reward_fn/std": 0.14185291826725005, + "grad_norm": 0.9515441656112671, + "kl": 0.049849181214813146, + "learning_rate": 4.953051643192488e-06, + "loss": 0.0019938603043556215, + "num_tokens": 62246.0, + "reward": 0.4975499987602234, + "reward_std": 0.1564827263355255, + "rewards/reward_fn/mean": 0.49755001068115234, + "rewards/reward_fn/std": 0.14618260115385057, "step": 15 }, { @@ -96,25 +96,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 28.25, + "completion_length": 26.525, "completions/clipped_ratio": 0.0, - "completions/max_length": 43.2, - "completions/max_terminated_length": 43.2, - "completions/mean_length": 28.25, - "completions/mean_terminated_length": 28.25, - "completions/min_length": 21.6, - "completions/min_terminated_length": 21.6, - "epoch": 0.09009009009009009, + "completions/max_length": 33.0, + "completions/max_terminated_length": 33.0, + "completions/mean_length": 26.525, + "completions/mean_terminated_length": 26.525, + "completions/min_length": 22.2, + "completions/min_terminated_length": 22.2, + "epoch": 0.08888888888888889, "frac_reward_zero_std": 0.0, - "grad_norm": 1.0063226222991943, - "kl": 0.0008983879492006963, - "learning_rate": 4.833333333333333e-06, - "loss": 3.593862056732178e-05, - "num_tokens": 82836.0, - "reward": 0.4881500005722046, - "reward_std": 0.14941166192293168, - "rewards/reward_fn/mean": 0.48814999461174013, - "rewards/reward_fn/std": 0.12791907638311387, + "grad_norm": 0.6862999200820923, + "kl": 0.04350169296376407, + "learning_rate": 4.835680751173709e-06, + "loss": 0.0017400771379470826, + "num_tokens": 81479.0, + "reward": 0.5159000098705292, + "reward_std": 0.17465537190437316, + "rewards/reward_fn/mean": 0.5158999919891357, + "rewards/reward_fn/std": 0.16455023884773254, "step": 20 }, { @@ -123,25 +123,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 27.25, + "completion_length": 28.0, "completions/clipped_ratio": 0.0, - "completions/max_length": 45.6, - "completions/max_terminated_length": 45.6, - "completions/mean_length": 27.25, - "completions/mean_terminated_length": 27.25, - "completions/min_length": 18.8, - "completions/min_terminated_length": 18.8, - "epoch": 0.11261261261261261, + "completions/max_length": 40.6, + "completions/max_terminated_length": 40.6, + "completions/mean_length": 28.0, + "completions/mean_terminated_length": 28.0, + "completions/min_length": 22.6, + "completions/min_terminated_length": 22.6, + "epoch": 0.1111111111111111, "frac_reward_zero_std": 0.0, - "grad_norm": 1.3132057189941406, - "kl": 0.003781939600594342, - "learning_rate": 4.714285714285715e-06, - "loss": 0.00015127062797546387, - "num_tokens": 103596.0, - "reward": 0.4680500030517578, - "reward_std": 0.17189764976501465, - "rewards/reward_fn/mean": 0.46804999113082885, - "rewards/reward_fn/std": 0.17421672195196153, + "grad_norm": 1.7516958713531494, + "kl": 0.05824573401478119, + "learning_rate": 4.71830985915493e-06, + "loss": 0.002329717576503754, + "num_tokens": 102833.0, + "reward": 0.5547999978065491, + "reward_std": 0.18328206837177277, + "rewards/reward_fn/mean": 0.5547999858856201, + "rewards/reward_fn/std": 0.1974634051322937, "step": 25 }, { @@ -150,25 +150,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 25.225, + "completion_length": 28.675, "completions/clipped_ratio": 0.0, - "completions/max_length": 34.6, - "completions/max_terminated_length": 34.6, - "completions/mean_length": 25.225, - "completions/mean_terminated_length": 25.225, - "completions/min_length": 19.2, - "completions/min_terminated_length": 19.2, - "epoch": 0.13513513513513514, + "completions/max_length": 37.4, + "completions/max_terminated_length": 37.4, + "completions/mean_length": 28.675, + "completions/mean_terminated_length": 28.675, + "completions/min_length": 22.2, + "completions/min_terminated_length": 22.2, + "epoch": 0.13333333333333333, "frac_reward_zero_std": 0.0, - "grad_norm": 1.5750545263290405, - "kl": 0.006673775642411783, - "learning_rate": 4.595238095238095e-06, - "loss": 0.0002669498324394226, - "num_tokens": 123651.0, - "reward": 0.48049998879432676, - "reward_std": 0.1656044065952301, - "rewards/reward_fn/mean": 0.48049998879432676, - "rewards/reward_fn/std": 0.13908967524766921, + "grad_norm": 1.5611271858215332, + "kl": 0.09752920938190072, + "learning_rate": 4.60093896713615e-06, + "loss": 0.0039011374115943908, + "num_tokens": 123876.0, + "reward": 0.5165499806404114, + "reward_std": 0.16765501499176025, + "rewards/reward_fn/mean": 0.5165499806404114, + "rewards/reward_fn/std": 0.16748485416173936, "step": 30 }, { @@ -177,25 +177,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 28.775, - "completions/clipped_ratio": 0.025, - "completions/max_length": 54.8, - "completions/max_terminated_length": 41.8, - "completions/mean_length": 28.775, - "completions/mean_terminated_length": 26.967857360839844, - "completions/min_length": 15.4, - "completions/min_terminated_length": 15.4, - "epoch": 0.15765765765765766, + "completion_length": 27.475, + "completions/clipped_ratio": 0.0, + "completions/max_length": 36.8, + "completions/max_terminated_length": 36.8, + "completions/mean_length": 27.475, + "completions/mean_terminated_length": 27.475, + "completions/min_length": 20.6, + "completions/min_terminated_length": 20.6, + "epoch": 0.15555555555555556, "frac_reward_zero_std": 0.0, - "grad_norm": 1.1105402708053589, - "kl": 0.005106111426721327, - "learning_rate": 4.476190476190477e-06, - "loss": 0.0002042025327682495, - "num_tokens": 144316.0, - "reward": 0.4778999924659729, - "reward_std": 0.17012988924980163, - "rewards/reward_fn/mean": 0.47790001034736634, - "rewards/reward_fn/std": 0.14147266000509262, + "grad_norm": 0.9550014138221741, + "kl": 0.08032013729680329, + "learning_rate": 4.483568075117371e-06, + "loss": 0.003212757408618927, + "num_tokens": 145019.0, + "reward": 0.5004499793052674, + "reward_std": 0.18391846716403962, + "rewards/reward_fn/mean": 0.5004500031471253, + "rewards/reward_fn/std": 0.16948954164981841, "step": 35 }, { @@ -204,25 +204,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 24.4, - "completions/clipped_ratio": 0.0, - "completions/max_length": 33.4, - "completions/max_terminated_length": 33.4, - "completions/mean_length": 24.4, - "completions/mean_terminated_length": 24.4, - "completions/min_length": 18.8, - "completions/min_terminated_length": 18.8, - "epoch": 0.18018018018018017, + "completion_length": 32.125, + "completions/clipped_ratio": 0.025, + "completions/max_length": 54.6, + "completions/max_terminated_length": 46.0, + "completions/mean_length": 32.125, + "completions/mean_terminated_length": 30.521428680419923, + "completions/min_length": 23.2, + "completions/min_terminated_length": 23.2, + "epoch": 0.17777777777777778, "frac_reward_zero_std": 0.0, - "grad_norm": 1.0676746368408203, - "kl": 0.006288998411764624, - "learning_rate": 4.357142857142857e-06, - "loss": 0.00025154203176498414, - "num_tokens": 164334.0, - "reward": 0.5014999985694886, - "reward_std": 0.15358359068632127, - "rewards/reward_fn/mean": 0.5014999985694886, - "rewards/reward_fn/std": 0.13824734836816788, + "grad_norm": 1.1203869581222534, + "kl": 0.08563535290304572, + "learning_rate": 4.3661971830985915e-06, + "loss": 0.003425435721874237, + "num_tokens": 164800.0, + "reward": 0.5472500026226044, + "reward_std": 0.19579786211252212, + "rewards/reward_fn/mean": 0.5472500085830688, + "rewards/reward_fn/std": 0.19857290089130403, "step": 40 }, { @@ -231,25 +231,25 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 27.35, + "completion_length": 27.875, "completions/clipped_ratio": 0.0, - "completions/max_length": 43.6, - "completions/max_terminated_length": 43.6, - "completions/mean_length": 27.35, - "completions/mean_terminated_length": 27.35, - "completions/min_length": 19.6, - "completions/min_terminated_length": 19.6, - "epoch": 0.20270270270270271, + "completions/max_length": 35.0, + "completions/max_terminated_length": 35.0, + "completions/mean_length": 27.875, + "completions/mean_terminated_length": 27.875, + "completions/min_length": 21.4, + "completions/min_terminated_length": 21.4, + "epoch": 0.2, "frac_reward_zero_std": 0.0, - "grad_norm": 1.0749404430389404, - "kl": 0.010629063473606948, - "learning_rate": 4.238095238095239e-06, - "loss": 0.0004251018166542053, - "num_tokens": 185140.0, - "reward": 0.524949985742569, - "reward_std": 0.14630039632320405, - "rewards/reward_fn/mean": 0.5249499917030335, - "rewards/reward_fn/std": 0.1460244983434677, + "grad_norm": 0.95042884349823, + "kl": 0.07295196709455923, + "learning_rate": 4.248826291079813e-06, + "loss": 0.0029180020093917845, + "num_tokens": 185761.0, + "reward": 0.5225999832153321, + "reward_std": 0.16489729881286622, + "rewards/reward_fn/mean": 0.5226000070571899, + "rewards/reward_fn/std": 0.16541497856378556, "step": 45 }, { @@ -258,31 +258,31 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "completion_length": 25.425, + "completion_length": 27.875, "completions/clipped_ratio": 0.0, "completions/max_length": 35.2, "completions/max_terminated_length": 35.2, - "completions/mean_length": 25.425, - "completions/mean_terminated_length": 25.425, - "completions/min_length": 17.6, - "completions/min_terminated_length": 17.6, - "epoch": 0.22522522522522523, + "completions/mean_length": 27.875, + "completions/mean_terminated_length": 27.875, + "completions/min_length": 21.4, + "completions/min_terminated_length": 21.4, + "epoch": 0.2222222222222222, "frac_reward_zero_std": 0.0, - "grad_norm": 1.4353065490722656, - "kl": 0.008290678875346203, - "learning_rate": 4.119047619047619e-06, - "loss": 0.0003315746784210205, - "num_tokens": 206569.0, - "reward": 0.4908999800682068, - "reward_std": 0.17055415213108063, - "rewards/reward_fn/mean": 0.49089999198913575, - "rewards/reward_fn/std": 0.154354290664196, + "grad_norm": 1.1037548780441284, + "kl": 0.08246680488809943, + "learning_rate": 4.131455399061034e-06, + "loss": 0.0032985761761665346, + "num_tokens": 206674.0, + "reward": 0.5245000004768372, + "reward_std": 0.1653215616941452, + "rewards/reward_fn/mean": 0.5245000004768372, + "rewards/reward_fn/std": 0.16877340227365495, "step": 50 } ], "logging_steps": 5, - "max_steps": 222, - "num_input_tokens_seen": 206569, + "max_steps": 225, + "num_input_tokens_seen": 206674, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { diff --git a/results/phase1/sft_warmup/README.md b/results/phase1/sft_warmup/README.md new file mode 100644 index 0000000000000000000000000000000000000000..dada6f76417a55ce66b9acf4e6aa3ebbf4881c01 --- /dev/null +++ b/results/phase1/sft_warmup/README.md @@ -0,0 +1,59 @@ +--- +base_model: unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit +library_name: transformers +model_name: sft_warmup +tags: +- generated_from_trainer +- unsloth +- trl +- sft +licence: license +--- + +# Model Card for sft_warmup + +This model is a fine-tuned version of [unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit](https://huggingface.co/unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + + + + +This model was trained with SFT. + +### Framework versions + +- TRL: 0.24.0 +- Transformers: 5.5.0 +- Pytorch: 2.10.0+cu128 +- Datasets: 4.3.0 +- Tokenizers: 0.22.2 + +## Citations + + + +Cite TRL as: + +```bibtex +@misc{vonwerra2022trl, + title = {{TRL: Transformer Reinforcement Learning}}, + author = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallou{\'e}dec}, + year = 2020, + journal = {GitHub repository}, + publisher = {GitHub}, + howpublished = {\url{https://github.com/huggingface/trl}} +} +``` \ No newline at end of file diff --git a/results/phase1/sft_warmup/checkpoint-48/README.md b/results/phase1/sft_warmup/checkpoint-48/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7aa58f41c9b1e6725818960c415ef5c16f75e4b5 --- /dev/null +++ b/results/phase1/sft_warmup/checkpoint-48/README.md @@ -0,0 +1,210 @@ +--- +base_model: unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit +- lora +- sft +- transformers +- trl +- unsloth +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/results/phase1/sft_warmup/checkpoint-48/adapter_config.json b/results/phase1/sft_warmup/checkpoint-48/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..d3cb3463c7f8c805e8cd8c25e64f3af87d6a00f3 --- /dev/null +++ b/results/phase1/sft_warmup/checkpoint-48/adapter_config.json @@ -0,0 +1,50 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": { + "base_model_class": "Qwen2ForCausalLM", + "parent_library": "transformers.models.qwen2.modeling_qwen2", + "unsloth_fixed": true + }, + "base_model_name_or_path": "unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.05, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "q_proj", + "k_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/results/phase1/sft_warmup/checkpoint-48/adapter_model.safetensors b/results/phase1/sft_warmup/checkpoint-48/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d533dd75d3e584e10bfe7244da18587937ab28a8 --- /dev/null +++ b/results/phase1/sft_warmup/checkpoint-48/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58986d6fe9c326b9899b549ea777f5f19819c15ce46e32313d9421d1af860335 +size 73911112 diff --git a/results/phase1/sft_warmup/checkpoint-48/chat_template.jinja b/results/phase1/sft_warmup/checkpoint-48/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..bdf7919a96cfe43d50914a007b9c0877bd0ec27e --- /dev/null +++ b/results/phase1/sft_warmup/checkpoint-48/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/results/phase1/sft_warmup/checkpoint-48/optimizer.pt b/results/phase1/sft_warmup/checkpoint-48/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..87818777c514fd67d683f42f69426572e18df7bd --- /dev/null +++ b/results/phase1/sft_warmup/checkpoint-48/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8612906f7146fa5cd7415c37909b407936414060b71ebc534a3f5ffaf7f3d53d +size 37969669 diff --git a/results/phase1/sft_warmup/checkpoint-48/rng_state.pth b/results/phase1/sft_warmup/checkpoint-48/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..0017ba39aef1f02c91ded317eba78e84c88a207c --- /dev/null +++ b/results/phase1/sft_warmup/checkpoint-48/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68f2e54b7fdd856296ab96e54e0e346f94b11770b6dd65da4ae4b576eb3a1cab +size 14645 diff --git a/results/phase1/sft_warmup/checkpoint-48/scaler.pt b/results/phase1/sft_warmup/checkpoint-48/scaler.pt new file mode 100644 index 0000000000000000000000000000000000000000..23567af93d476da676c1f91239a4fd55416a4421 --- /dev/null +++ b/results/phase1/sft_warmup/checkpoint-48/scaler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f58cd4ee5f6d68ac6fd437e159c5ad7c78bf0f0800bb6f77185bf13e42a508c +size 1383 diff --git a/results/phase1/sft_warmup/checkpoint-48/scheduler.pt b/results/phase1/sft_warmup/checkpoint-48/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..7a697a64b39f7cf420eda266909e6c7bc476dfbb --- /dev/null +++ b/results/phase1/sft_warmup/checkpoint-48/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f8c15637bdf0a16f4962ec44b7231f03100e3eae1261ae580198c04915f691c +size 1465 diff --git a/results/phase1/sft_warmup/checkpoint-48/tokenizer.json b/results/phase1/sft_warmup/checkpoint-48/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..5340d8195cfed687e080acf4f7cfdc46d18d5924 --- /dev/null +++ b/results/phase1/sft_warmup/checkpoint-48/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd5948af71b4f56cf697f7580814c7ce8b80595ef985544efcacf716126a2e31 +size 11422356 diff --git a/results/phase1/sft_warmup/checkpoint-48/tokenizer_config.json b/results/phase1/sft_warmup/checkpoint-48/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..d3dac539765625f7e736bb2cdacac98bfa616b1f --- /dev/null +++ b/results/phase1/sft_warmup/checkpoint-48/tokenizer_config.json @@ -0,0 +1,201 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "is_local": false, + "model_max_length": 32768, + "pad_token": "<|PAD_TOKEN|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151657": { + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151658": { + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151665": { + "content": "<|PAD_TOKEN|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + } + } +} diff --git a/results/phase1/sft_warmup/checkpoint-48/trainer_state.json b/results/phase1/sft_warmup/checkpoint-48/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..1eda33f9e1a59d1e94cbbd64905d24cb845ea9da --- /dev/null +++ b/results/phase1/sft_warmup/checkpoint-48/trainer_state.json @@ -0,0 +1,97 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 48, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.20833333333333334, + "grad_norm": 1.8928676843643188, + "learning_rate": 1.9555555555555557e-05, + "loss": 2.413274383544922, + "step": 5 + }, + { + "epoch": 0.4166666666666667, + "grad_norm": 1.190262794494629, + "learning_rate": 1.7333333333333336e-05, + "loss": 2.1639257431030274, + "step": 10 + }, + { + "epoch": 0.625, + "grad_norm": 0.9186313152313232, + "learning_rate": 1.5111111111111112e-05, + "loss": 1.9978204727172852, + "step": 15 + }, + { + "epoch": 0.8333333333333334, + "grad_norm": 0.8808704018592834, + "learning_rate": 1.288888888888889e-05, + "loss": 2.0219154357910156, + "step": 20 + }, + { + "epoch": 1.0416666666666667, + "grad_norm": 0.9148248434066772, + "learning_rate": 1.0666666666666667e-05, + "loss": 2.199655532836914, + "step": 25 + }, + { + "epoch": 1.25, + "grad_norm": 1.0190837383270264, + "learning_rate": 8.444444444444446e-06, + "loss": 1.923904037475586, + "step": 30 + }, + { + "epoch": 1.4583333333333333, + "grad_norm": 1.0638080835342407, + "learning_rate": 6.222222222222223e-06, + "loss": 1.856580924987793, + "step": 35 + }, + { + "epoch": 1.6666666666666665, + "grad_norm": 1.0227775573730469, + "learning_rate": 4.000000000000001e-06, + "loss": 1.7232439041137695, + "step": 40 + }, + { + "epoch": 1.875, + "grad_norm": 1.1590324640274048, + "learning_rate": 1.777777777777778e-06, + "loss": 1.6328853607177733, + "step": 45 + } + ], + "logging_steps": 5, + "max_steps": 48, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 1532234848622592.0, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/results/phase1/sft_warmup/checkpoint-48/training_args.bin b/results/phase1/sft_warmup/checkpoint-48/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..17e4f045c4e83b9cf095d3c39c59b767823a96ca --- /dev/null +++ b/results/phase1/sft_warmup/checkpoint-48/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:640657cb40666a08fedb1b44ce24e15ac5ce09c53ef4e83d41df7b6d033a4cea +size 5713 diff --git a/train_llm.py b/train_llm.py index 85834374dd13708e42fbf6c2c3f168a934d925c4..2a4e2fa886bd507d16ee1be03a66fd5a46356913 100644 --- a/train_llm.py +++ b/train_llm.py @@ -339,12 +339,11 @@ def _parse_action(text) -> dict: Parse LLM completion → action dict. Takes the LAST JSON object in the text (handles chain-of-thought prefix). - FIX: TRL ≥0.9 (used with Unsloth 2026.x) passes completions as - list[dict] in chat-message format instead of a plain string. - e.g. [{"role": "assistant", "content": '{"action_type":"assign"...}'}] - We extract the assistant content string before any string operations. + FIX: TRL >=0.9 / Unsloth 2026.x passes completions as list[dict] in chat + message format instead of a plain str: + e.g. [{"role": "assistant", "content": '{"action_type":"assign"...}'}] + Extract the assistant content string before any string operations. """ - # Handle TRL ≥0.9 list[dict] format if isinstance(text, list): text = " ".join( m.get("content", "") for m in text if m.get("role") == "assistant" @@ -863,6 +862,12 @@ def run_sft(model, tokenizer, phase: str, n_examples: int, output_dir: str): return {"text": "\n".join(parts)} sft_data = sft_data.map(format_fn) + # FIX: Unsloth SFTTrainer (2026.x) detects "prompt"+"completion" columns + # and routes to _tokenize_pc which does list+str -> TypeError. + # Drop them so SFTTrainer only sees "text" and uses dataset_text_field path. + cols_to_drop = [c for c in ["prompt", "completion"] if c in sft_data.column_names] + if cols_to_drop: + sft_data = sft_data.remove_columns(cols_to_drop) sft_dir = str(Path(output_dir) / "sft_warmup") sft_conf = SFTConfig( diff --git a/trained_model.zip b/trained_model.zip index e59509e8a53d3e65db4fe3441a471ce7a9bc4ce2..67d002e3d8dfa8ce7e35d83f25f558cebc8467c0 100644 --- a/trained_model.zip +++ b/trained_model.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:24d0ff3bdc8c555c008cfbabc7724d0b1fdad7a7b0dbf8165f03bb4858d54a68 -size 590921989 +oid sha256:9b3e2862b95394e74e69db98c03b198042e10b43fccaee7f582f1b8ab6199ff6 +size 692060689