Hugging Face's logo

kishan51
/
llm-zero-lite-experiments

Reinforcement Learning
PEFT
Safetensors
grpo
countdown
llm-controller
qwen3
Model card Files
xet
 Community
kishan51 commited on
Commit
4f99f73
·
verified ·
1 Parent(s): 6ac3df6

Add files using upload-large-folder tool

Browse files
Files changed (50) hide show
  1. README.md +28 -0
  2. artifact_manifest.json +0 -0
  3. configs/base.yaml +38 -0
  4. requirements.txt +13 -0
  5. runs/_base_eval_qwen3-1p7b_n3_eval200_k4_seed42/eval_samples.jsonl +0 -0
  6. runs/completion_length_by_stage.png +0 -0
  7. runs/continuous_long_seed42.log +0 -0
  8. runs/continuous_long_seed42/experiment_config.json +39 -0
  9. runs/continuous_long_seed42/history.csv +7 -0
  10. runs/continuous_long_seed42/train_log.jsonl +0 -0
  11. runs/continuous_seed42.log +0 -0
  12. runs/continuous_seed42/baseline_metrics.json +18 -0
  13. runs/continuous_seed42/experiment_config.json +39 -0
  14. runs/continuous_seed42/train_log.jsonl +0 -0
  15. runs/kl_by_stage.png +0 -0
  16. runs/llm_long_seed42/baseline_metrics.json +18 -0
  17. runs/llm_long_seed42/experiment_config.json +39 -0
  18. runs/llm_seed42.log +0 -0
  19. runs/llm_seed42/experiment_config.json +39 -0
  20. runs/llm_seed42/history.csv +3 -0
  21. runs/llm_seed42/stage_0/checkpoint/added_tokens.json +28 -0
  22. runs/llm_seed42/stage_0/config.json +39 -0
  23. runs/llm_seed42/stage_0/decision.json +12 -0
  24. runs/llm_seed42/stage_0/eval_samples.jsonl +0 -0
  25. runs/llm_seed42/stage_0/metrics.json +28 -0
  26. runs/llm_seed42/stage_0/train_log.jsonl +0 -0
  27. runs/llm_seed42/stage_1/config.json +39 -0
  28. runs/smoke_continuous.log +154 -0
  29. runs/smoke_continuous/baseline_metrics.json +18 -0
  30. runs/smoke_continuous/experiment_config.json +38 -0
  31. runs/smoke_continuous/history.csv +4 -0
  32. runs/smoke_continuous/train_log.jsonl +41 -0
  33. runs/smoke_staged.log +211 -0
  34. runs/smoke_staged/baseline_metrics.json +18 -0
  35. runs/smoke_staged/history.csv +3 -0
  36. runs/staged_fixed_long_seed42.log +0 -0
  37. runs/staged_fixed_long_seed42/baseline_metrics.json +18 -0
  38. runs/staged_fixed_seed42/baseline_metrics.json +18 -0
  39. runs/staged_fixed_seed42/experiment_config.json +39 -0
  40. runs/staged_fixed_seed42/history.csv +3 -0
  41. runs/summary_table.csv +34 -0
  42. src/__init__.py +1 -0
  43. src/controllers.py +202 -0
  44. src/data.py +51 -0
  45. src/evaluate.py +102 -0
  46. src/logging_utils.py +51 -0
  47. src/plot_results.py +44 -0
  48. src/rewards.py +151 -0
  49. src/run_experiment.py +218 -0
  50. src/train_stage.py +178 -0
README.md ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ base_model: Qwen/Qwen3-1.7B
3
+ library_name: peft
4
+ license: apache-2.0
5
+ tags:
6
+ - grpo
7
+ - reinforcement-learning
8
+ - countdown
9
+ - llm-controller
10
+ - qwen3
11
+ ---
12
+
13
+ # LLM-Zero-Lite Experiments
14
+
15
+ A controlled comparison of continuous GRPO, fixed staged GRPO, and an
16
+ LLM-controlled staged GRPO schedule on three-number Countdown using
17
+ `Qwen/Qwen3-1.7B` with LoRA.
18
+
19
+ ## Final 1,000-step results
20
+
21
+ | Method | Greedy accuracy | Sampled pass@1 | Sampled pass@4 |
22
+ |---|---:|---:|---:|
23
+ | Continuous GRPO | 26.5% | 31.0% | 35.5% |
24
+ | Fixed staged GRPO | 34.5% | 34.5% | 39.5% |
25
+ | LLM controller | **36.5%** | **37.5%** | **40.5%** |
26
+
27
+ The `runs/` directory contains metrics, evaluation samples, configuration
28
+ history, controller decisions, logs, plots, and all saved LoRA checkpoints.
artifact_manifest.json ADDED
The diff for this file is too large to render. See raw diff
 
configs/base.yaml ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ model_name: Qwen/Qwen3-1.7B
3
+ train_size: 1000
4
+ eval_size: 200
5
+ num_numbers: 3
6
+ disable_thinking: true
7
+ num_stages: 5
8
+ steps_per_stage: 200
9
+ seed: 42
10
+ learning_rate: 1.0e-6
11
+ lr_scheduler_type: cosine
12
+ warmup_ratio: 0.03
13
+ beta: 0.04
14
+ temperature: 1.0
15
+ max_prompt_length: 256
16
+ max_completion_length: 512
17
+ num_generations: 8
18
+ per_device_train_batch_size: 32
19
+ gradient_accumulation_steps: 1
20
+ use_vllm: true
21
+ vllm_mode: colocate
22
+ vllm_gpu_memory_utilization: 0.55
23
+ vllm_enable_sleep_mode: false
24
+ vllm_importance_sampling_correction: true
25
+ vllm_max_model_length: 768
26
+ vllm_tensor_parallel_size: 1
27
+ eval_batch_size: 64
28
+ eval_num_samples: 4
29
+ eval_temperature: 1.0
30
+ lora_r: 32
31
+ lora_alpha: 64
32
+ lora_dropout: 0.05
33
+ wandb_project: llm-zero-lite
34
+ llm_controller_mode: api
35
+ llm_controller_fail_open: true
36
+ llm_controller_max_retries: 3
37
+ llm_controller_max_tokens: 1200
38
+ llm_controller_model: z-ai/glm-5.2
requirements.txt ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ trl[vllm]==0.29.1
2
+ vllm>=0.12.0,<0.13.0
3
+ transformers>=4.56,<5
4
+ peft==0.17.1
5
+ datasets>=3.0
6
+ accelerate>=1.2
7
+ pandas
8
+ matplotlib
9
+ pyyaml
10
+ openai
11
+ wandb
12
+ pytest
13
+ tabulate
runs/_base_eval_qwen3-1p7b_n3_eval200_k4_seed42/eval_samples.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
runs/completion_length_by_stage.png ADDED
runs/continuous_long_seed42.log ADDED
The diff for this file is too large to render. See raw diff
 
runs/continuous_long_seed42/experiment_config.json ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_name": "Qwen/Qwen3-1.7B",
3
+ "train_size": 1000,
4
+ "eval_size": 200,
5
+ "num_numbers": 3,
6
+ "disable_thinking": true,
7
+ "num_stages": 5,
8
+ "steps_per_stage": 200,
9
+ "seed": 42,
10
+ "learning_rate": 1e-06,
11
+ "lr_scheduler_type": "cosine",
12
+ "warmup_ratio": 0.03,
13
+ "beta": 0.04,
14
+ "temperature": 1.0,
15
+ "max_prompt_length": 256,
16
+ "max_completion_length": 512,
17
+ "num_generations": 8,
18
+ "per_device_train_batch_size": 32,
19
+ "gradient_accumulation_steps": 1,
20
+ "use_vllm": true,
21
+ "vllm_mode": "colocate",
22
+ "vllm_gpu_memory_utilization": 0.55,
23
+ "vllm_enable_sleep_mode": false,
24
+ "vllm_importance_sampling_correction": true,
25
+ "vllm_max_model_length": 768,
26
+ "vllm_tensor_parallel_size": 1,
27
+ "eval_batch_size": 64,
28
+ "eval_num_samples": 4,
29
+ "eval_temperature": 1.0,
30
+ "lora_r": 32,
31
+ "lora_alpha": 64,
32
+ "lora_dropout": 0.05,
33
+ "wandb_project": "llm-zero-lite",
34
+ "llm_controller_mode": "api",
35
+ "llm_controller_fail_open": true,
36
+ "llm_controller_max_retries": 3,
37
+ "llm_controller_max_tokens": 1200,
38
+ "llm_controller_model": "z-ai/glm-5.2"
39
+ }
runs/continuous_long_seed42/history.csv ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ method,run_name,stage,global_train_steps,eval_accuracy,eval_greedy_accuracy,eval_sampled_pass_at_1,eval_sampled_pass_at_4,eval_avg_completion_length,eval_num_samples,eval_temperature,learning_rate,beta,temperature,max_completion_length,num_generations,train_reward_mean,train_reward_std,kl_mean,entropy_mean,avg_completion_length,completion_length_clip_ratio,grad_norm,last_loss,end_learning_rate,wall_clock_seconds
2
+ continuous_grpo,continuous_long_seed42,-1,0,0.225,0.225,0.22,0.31,19.185,4,1.0,1e-06,0.04,1.0,512,8,,,,,,,,,,
3
+ continuous_grpo,continuous_long_seed42,0,200,0.255,0.255,0.245,0.325,17.88,4,1.0,1e-06,0.04,1.0,512,8,0.5683742192387581,0.4761904725804925,0.019905953802372096,0.0570064028352499,19.43875,0.0,2.534748571543023,-0.0074,9.269534157077176e-07,736.0401890277863
4
+ continuous_grpo,continuous_long_seed42,1,400,0.25,0.25,0.255,0.34,17.585,4,1.0,1e-06,0.04,1.0,512,8,0.6293063132464886,0.4732742319442332,0.05787532195914537,0.048856847565621135,18.8446875,0.0,2.588927525356412,-0.0353,6.834598583458861e-07,736.0401890277863
5
+ continuous_grpo,continuous_long_seed42,2,600,0.26,0.26,0.275,0.345,17.85,4,1.0,1e-06,0.04,1.0,512,8,0.6279417244344949,0.47874392770696433,0.08111654391395859,0.04761551594827324,18.9290625,0.0,2.8439276323188096,-0.0248,3.656440744218464e-07,736.0401890277863
6
+ continuous_grpo,continuous_long_seed42,3,800,0.255,0.255,0.29,0.34,17.79,4,1.0,1e-06,0.04,1.0,512,8,0.636554714217782,0.49074650494381783,0.0867256929987343,0.050090711209923026,19.14921875,0.0,3.523692013407126,-0.0905,1.0225780783651689e-07,736.0401890277863
7
+ continuous_grpo,continuous_long_seed42,4,1000,0.265,0.265,0.31,0.355,17.83,4,1.0,1e-06,0.04,1.0,512,8,0.6424943773448467,0.4787482109293342,0.07954302004072815,0.046858076183125374,19.0875,0.0,2.7047918717749417,0.0519,2.6223817020665227e-12,736.0401890277863
runs/continuous_long_seed42/train_log.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
runs/continuous_seed42.log ADDED
The diff for this file is too large to render. See raw diff
 
runs/continuous_seed42/baseline_metrics.json ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "method": "continuous_grpo",
3
+ "run_name": "continuous_seed42",
4
+ "stage": -1,
5
+ "global_train_steps": 0,
6
+ "eval_accuracy": 0.225,
7
+ "eval_greedy_accuracy": 0.225,
8
+ "eval_sampled_pass_at_1": 0.22,
9
+ "eval_sampled_pass_at_4": 0.31,
10
+ "eval_avg_completion_length": 19.185,
11
+ "eval_num_samples": 4,
12
+ "eval_temperature": 1.0,
13
+ "learning_rate": 1e-06,
14
+ "beta": 0.04,
15
+ "temperature": 1.0,
16
+ "max_completion_length": 512,
17
+ "num_generations": 16
18
+ }
runs/continuous_seed42/experiment_config.json ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_name": "Qwen/Qwen3-1.7B",
3
+ "train_size": 1000,
4
+ "eval_size": 200,
5
+ "num_numbers": 3,
6
+ "disable_thinking": true,
7
+ "num_stages": 5,
8
+ "steps_per_stage": 100,
9
+ "seed": 42,
10
+ "learning_rate": 1e-06,
11
+ "lr_scheduler_type": "cosine",
12
+ "warmup_ratio": 0.03,
13
+ "beta": 0.04,
14
+ "temperature": 1.0,
15
+ "max_prompt_length": 256,
16
+ "max_completion_length": 512,
17
+ "num_generations": 16,
18
+ "per_device_train_batch_size": 32,
19
+ "gradient_accumulation_steps": 1,
20
+ "use_vllm": true,
21
+ "vllm_mode": "colocate",
22
+ "vllm_gpu_memory_utilization": 0.55,
23
+ "vllm_enable_sleep_mode": false,
24
+ "vllm_importance_sampling_correction": true,
25
+ "vllm_max_model_length": 768,
26
+ "vllm_tensor_parallel_size": 1,
27
+ "eval_batch_size": 64,
28
+ "eval_num_samples": 4,
29
+ "eval_temperature": 1.0,
30
+ "lora_r": 32,
31
+ "lora_alpha": 64,
32
+ "lora_dropout": 0.05,
33
+ "wandb_project": "llm-zero-lite",
34
+ "llm_controller_mode": "api",
35
+ "llm_controller_fail_open": true,
36
+ "llm_controller_max_retries": 3,
37
+ "llm_controller_max_tokens": 1200,
38
+ "llm_controller_model": "z-ai/glm-5.2"
39
+ }
runs/continuous_seed42/train_log.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
runs/kl_by_stage.png ADDED
runs/llm_long_seed42/baseline_metrics.json ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "method": "llm_controller",
3
+ "run_name": "llm_long_seed42",
4
+ "stage": -1,
5
+ "global_train_steps": 0,
6
+ "eval_accuracy": 0.225,
7
+ "eval_greedy_accuracy": 0.225,
8
+ "eval_sampled_pass_at_1": 0.22,
9
+ "eval_sampled_pass_at_4": 0.31,
10
+ "eval_avg_completion_length": 19.185,
11
+ "eval_num_samples": 4,
12
+ "eval_temperature": 1.0,
13
+ "learning_rate": 1e-06,
14
+ "beta": 0.04,
15
+ "temperature": 1.0,
16
+ "max_completion_length": 512,
17
+ "num_generations": 8
18
+ }
runs/llm_long_seed42/experiment_config.json ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_name": "Qwen/Qwen3-1.7B",
3
+ "train_size": 1000,
4
+ "eval_size": 200,
5
+ "num_numbers": 3,
6
+ "disable_thinking": true,
7
+ "num_stages": 5,
8
+ "steps_per_stage": 200,
9
+ "seed": 42,
10
+ "learning_rate": 1e-06,
11
+ "lr_scheduler_type": "cosine",
12
+ "warmup_ratio": 0.03,
13
+ "beta": 0.04,
14
+ "temperature": 1.0,
15
+ "max_prompt_length": 256,
16
+ "max_completion_length": 512,
17
+ "num_generations": 8,
18
+ "per_device_train_batch_size": 32,
19
+ "gradient_accumulation_steps": 1,
20
+ "use_vllm": true,
21
+ "vllm_mode": "colocate",
22
+ "vllm_gpu_memory_utilization": 0.55,
23
+ "vllm_enable_sleep_mode": false,
24
+ "vllm_importance_sampling_correction": true,
25
+ "vllm_max_model_length": 768,
26
+ "vllm_tensor_parallel_size": 1,
27
+ "eval_batch_size": 64,
28
+ "eval_num_samples": 4,
29
+ "eval_temperature": 1.0,
30
+ "lora_r": 32,
31
+ "lora_alpha": 64,
32
+ "lora_dropout": 0.05,
33
+ "wandb_project": "llm-zero-lite",
34
+ "llm_controller_mode": "api",
35
+ "llm_controller_fail_open": true,
36
+ "llm_controller_max_retries": 3,
37
+ "llm_controller_max_tokens": 1200,
38
+ "llm_controller_model": "z-ai/glm-5.2"
39
+ }
runs/llm_seed42.log ADDED
The diff for this file is too large to render. See raw diff
 
runs/llm_seed42/experiment_config.json ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_name": "Qwen/Qwen3-1.7B",
3
+ "train_size": 1000,
4
+ "eval_size": 200,
5
+ "num_numbers": 3,
6
+ "disable_thinking": true,
7
+ "num_stages": 5,
8
+ "steps_per_stage": 100,
9
+ "seed": 42,
10
+ "learning_rate": 1e-06,
11
+ "lr_scheduler_type": "cosine",
12
+ "warmup_ratio": 0.03,
13
+ "beta": 0.04,
14
+ "temperature": 1.0,
15
+ "max_prompt_length": 256,
16
+ "max_completion_length": 512,
17
+ "num_generations": 16,
18
+ "per_device_train_batch_size": 32,
19
+ "gradient_accumulation_steps": 1,
20
+ "use_vllm": true,
21
+ "vllm_mode": "colocate",
22
+ "vllm_gpu_memory_utilization": 0.55,
23
+ "vllm_enable_sleep_mode": false,
24
+ "vllm_importance_sampling_correction": true,
25
+ "vllm_max_model_length": 768,
26
+ "vllm_tensor_parallel_size": 1,
27
+ "eval_batch_size": 64,
28
+ "eval_num_samples": 4,
29
+ "eval_temperature": 1.0,
30
+ "lora_r": 32,
31
+ "lora_alpha": 64,
32
+ "lora_dropout": 0.05,
33
+ "wandb_project": "llm-zero-lite",
34
+ "llm_controller_mode": "api",
35
+ "llm_controller_fail_open": true,
36
+ "llm_controller_max_retries": 3,
37
+ "llm_controller_max_tokens": 1200,
38
+ "llm_controller_model": "z-ai/glm-5.2"
39
+ }
runs/llm_seed42/history.csv ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ method,run_name,stage,global_train_steps,eval_accuracy,eval_greedy_accuracy,eval_sampled_pass_at_1,eval_sampled_pass_at_4,eval_avg_completion_length,eval_num_samples,eval_temperature,learning_rate,beta,temperature,max_completion_length,num_generations,train_reward_mean,train_reward_std,kl_mean,entropy_mean,avg_completion_length,completion_length_clip_ratio,grad_norm,last_loss,end_learning_rate,wall_clock_seconds
2
+ llm_controller,llm_seed42,-1,0,0.225,0.225,0.22,0.31,19.185,4,1.0,1e-06,0.04,1.0,512,16,,,,,,,,,,
3
+ llm_controller,llm_seed42,0,100,0.24,0.24,0.215,0.31,18.855,4,1.0,1e-06,0.04,1.0,512,16,0.5534133762121201,0.3936914769653231,0.0038538010309275704,0.05732283437624574,19.9303125,0.0,3.1614123660698534,0.102,2.6221547724253333e-10,78.17542266845703
runs/llm_seed42/stage_0/checkpoint/added_tokens.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "</think>": 151668,
3
+ "</tool_call>": 151658,
4
+ "</tool_response>": 151666,
5
+ "<think>": 151667,
6
+ "<tool_call>": 151657,
7
+ "<tool_response>": 151665,
8
+ "<|box_end|>": 151649,
9
+ "<|box_start|>": 151648,
10
+ "<|endoftext|>": 151643,
11
+ "<|file_sep|>": 151664,
12
+ "<|fim_middle|>": 151660,
13
+ "<|fim_pad|>": 151662,
14
+ "<|fim_prefix|>": 151659,
15
+ "<|fim_suffix|>": 151661,
16
+ "<|im_end|>": 151645,
17
+ "<|im_start|>": 151644,
18
+ "<|image_pad|>": 151655,
19
+ "<|object_ref_end|>": 151647,
20
+ "<|object_ref_start|>": 151646,
21
+ "<|quad_end|>": 151651,
22
+ "<|quad_start|>": 151650,
23
+ "<|repo_name|>": 151663,
24
+ "<|video_pad|>": 151656,
25
+ "<|vision_end|>": 151653,
26
+ "<|vision_pad|>": 151654,
27
+ "<|vision_start|>": 151652
28
+ }
runs/llm_seed42/stage_0/config.json ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_name": "Qwen/Qwen3-1.7B",
3
+ "train_size": 1000,
4
+ "eval_size": 200,
5
+ "num_numbers": 3,
6
+ "disable_thinking": true,
7
+ "num_stages": 5,
8
+ "steps_per_stage": 100,
9
+ "seed": 42,
10
+ "learning_rate": 1e-06,
11
+ "lr_scheduler_type": "cosine",
12
+ "warmup_ratio": 0.03,
13
+ "beta": 0.04,
14
+ "temperature": 1.0,
15
+ "max_prompt_length": 256,
16
+ "max_completion_length": 512,
17
+ "num_generations": 16,
18
+ "per_device_train_batch_size": 32,
19
+ "gradient_accumulation_steps": 1,
20
+ "use_vllm": true,
21
+ "vllm_mode": "colocate",
22
+ "vllm_gpu_memory_utilization": 0.55,
23
+ "vllm_enable_sleep_mode": false,
24
+ "vllm_importance_sampling_correction": true,
25
+ "vllm_max_model_length": 768,
26
+ "vllm_tensor_parallel_size": 1,
27
+ "eval_batch_size": 64,
28
+ "eval_num_samples": 4,
29
+ "eval_temperature": 1.0,
30
+ "lora_r": 32,
31
+ "lora_alpha": 64,
32
+ "lora_dropout": 0.05,
33
+ "wandb_project": "llm-zero-lite",
34
+ "llm_controller_mode": "api",
35
+ "llm_controller_fail_open": true,
36
+ "llm_controller_max_retries": 3,
37
+ "llm_controller_max_tokens": 1200,
38
+ "llm_controller_model": "z-ai/glm-5.2"
39
+ }
runs/llm_seed42/stage_0/decision.json ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "learning_rate_multiplier": 1.5,
3
+ "beta_multiplier": 1.0,
4
+ "temperature_delta": 0.1,
5
+ "max_completion_length_delta": 0,
6
+ "num_generations_delta": 0,
7
+ "early_stop": false,
8
+ "rollback_to_best_checkpoint": false,
9
+ "reason": "Entropy collapse (0.057) and near-zero effective LR (2.6e-10) have stalled learning. Eval accuracy barely improved. Raising LR 1.5x to restore gradient signal and temperature +0.1 to break entropy collapse and encourage exploration. Beta kept low at 0.04 to allow divergence from reference.",
10
+ "controller_analysis": "## Analysis\n\n**Reward & Accuracy:** Train reward is 0.553 with high std (0.394), but eval accuracy barely moved (0.225 \u2192 0.24). Pass@4 remains 0.31, unchanged. This suggests the model is exploiting reward signal on training data without genuine generalization.\n\n**Entropy Collapse:** The most critical signal is entropy at 0.057 \u2014 extremely low. The model has nearly collapsed to deterministic outputs despite temperature 1.0. This severely limits exploration and explains the stagnant eval metrics. The model is essentially memorizing short patterns rather than learning diverse reasoning strategies.\n\n**KL & LR:** KL is 0.0039, meaning the policy has barely diverged from the reference model. The end_learning_rate (2.6e-10) indicates the cosine schedule has fully decayed, so the model is effectively no longer learning. The combination of low entropy, low KL, and near-zero effective LR means the model is stuck.\n\n**Completion Length:** At ~20 tokens, outputs are very short. While no clipping occurs, the brevity suggests the model isn't engaging in extended reasoning that could improve accuracy.\n\n**Tradeoffs:** Increasing temperature is essential to break the entropy collapse and restore exploration. Raising the LR multiplier counteracts the fully-decayed schedule. Keeping beta low (0.04) allows the policy to diverge more freely. There's risk that higher temperature increases reward variance, but the current stagnation demands it \u2014 the model cannot improve if it never explores.\n\n**Decision:** Increase LR (1.5\u00d7) to restore learning signal, increase temperature (+0.1) to combat entropy collapse, keep beta unchanged.",
11
+ "controller_attempts": 2
12
+ }
runs/llm_seed42/stage_0/eval_samples.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
runs/llm_seed42/stage_0/metrics.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "method": "llm_controller",
3
+ "run_name": "llm_seed42",
4
+ "stage": 0,
5
+ "global_train_steps": 100,
6
+ "train_reward_mean": 0.5534133762121201,
7
+ "train_reward_std": 0.3936914769653231,
8
+ "kl_mean": 0.0038538010309275704,
9
+ "entropy_mean": 0.05732283437624574,
10
+ "avg_completion_length": 19.9303125,
11
+ "completion_length_clip_ratio": 0.0,
12
+ "grad_norm": 3.1614123660698534,
13
+ "last_loss": 0.102,
14
+ "end_learning_rate": 2.6221547724253333e-10,
15
+ "wall_clock_seconds": 78.17542266845703,
16
+ "eval_accuracy": 0.24,
17
+ "eval_greedy_accuracy": 0.24,
18
+ "eval_sampled_pass_at_1": 0.215,
19
+ "eval_sampled_pass_at_4": 0.31,
20
+ "eval_avg_completion_length": 18.855,
21
+ "eval_num_samples": 4,
22
+ "eval_temperature": 1.0,
23
+ "learning_rate": 1e-06,
24
+ "beta": 0.04,
25
+ "temperature": 1.0,
26
+ "max_completion_length": 512,
27
+ "num_generations": 16
28
+ }
runs/llm_seed42/stage_0/train_log.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
runs/llm_seed42/stage_1/config.json ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_name": "Qwen/Qwen3-1.7B",
3
+ "train_size": 1000,
4
+ "eval_size": 200,
5
+ "num_numbers": 3,
6
+ "disable_thinking": true,
7
+ "num_stages": 5,
8
+ "steps_per_stage": 100,
9
+ "seed": 42,
10
+ "learning_rate": 1.5e-06,
11
+ "lr_scheduler_type": "cosine",
12
+ "warmup_ratio": 0.03,
13
+ "beta": 0.04,
14
+ "temperature": 1.1,
15
+ "max_prompt_length": 256,
16
+ "max_completion_length": 512,
17
+ "num_generations": 16,
18
+ "per_device_train_batch_size": 32,
19
+ "gradient_accumulation_steps": 1,
20
+ "use_vllm": true,
21
+ "vllm_mode": "colocate",
22
+ "vllm_gpu_memory_utilization": 0.55,
23
+ "vllm_enable_sleep_mode": false,
24
+ "vllm_importance_sampling_correction": true,
25
+ "vllm_max_model_length": 768,
26
+ "vllm_tensor_parallel_size": 1,
27
+ "eval_batch_size": 64,
28
+ "eval_num_samples": 4,
29
+ "eval_temperature": 1.0,
30
+ "lora_r": 32,
31
+ "lora_alpha": 64,
32
+ "lora_dropout": 0.05,
33
+ "wandb_project": "llm-zero-lite",
34
+ "llm_controller_mode": "api",
35
+ "llm_controller_fail_open": true,
36
+ "llm_controller_max_retries": 3,
37
+ "llm_controller_max_tokens": 1200,
38
+ "llm_controller_model": "z-ai/glm-5.2"
39
+ }
runs/smoke_continuous.log ADDED
@@ -0,0 +1,154 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+
3
+
4
+
5
+
6
+ 2026-06-21 10:42:43,386 - INFO - autotuner.py:256 - flashinfer.jit: [Autotuner]: Autotuning process starts ...
7
+ 2026-06-21 10:42:43,392 - INFO - autotuner.py:262 - flashinfer.jit: [Autotuner]: Autotuning process ends
8
+
9
+
10
+ wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
11
+ wandb: Currently logged in as: kishanvavdara to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
12
+ wandb: Tracking run with wandb version 0.27.2
13
+ wandb: Run data is saved locally in /content/llm-zero-lite/wandb/run-20260621_104246-1iawvj3t
14
+ wandb: Run `wandb offline` to turn off syncing.
15
+ wandb: Syncing run smoke_continuous_continuous
16
+ wandb: ⭐️ View project at https://wandb.ai/kishanvavdara/llm-zero-lite-smoke
17
+ wandb: 🚀 View run at https://wandb.ai/kishanvavdara/llm-zero-lite-smoke/runs/1iawvj3t
18
+ wandb: Detected [openai] in use.
19
+ wandb: Use W&B Weave for improved LLM call tracing. Install Weave with `pip install weave` then add `import weave` to the top of your script.
20
+ wandb: For more information, check out the docs at: https://weave-docs.wandb.ai
21
+ {'loss': 0.0109, 'grad_norm': 6.041065692901611, 'learning_rate': 0.0, 'num_tokens': 3425.0, 'completions/mean_length': 19.53125, 'completions/min_length': 18.0, 'completions/max_length': 30.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 19.53125, 'completions/min_terminated_length': 18.0, 'completions/max_terminated_length': 30.0, 'rewards/countdown_reward/mean': 0.59375, 'rewards/countdown_reward/std': 0.49899089336395264, 'rewards/format_reward/mean': 0.09375, 'rewards/format_reward/std': 0.024593466892838478, 'rewards/valid_numbers_reward/mean': 0.1875, 'rewards/valid_numbers_reward/std': 0.049186933785676956, 'rewards/proximity_reward/mean': 0.13208332657814026, 'rewards/proximity_reward/std': 0.08473340421915054, 'reward': 1.0070834159851074, 'reward_std': 0.6107084155082703, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.014617112465202808, 'sampling/sampling_logp_difference/max': 0.44372355937957764, 'sampling/importance_sampling_ratio/min': 0.4675312638282776, 'sampling/importance_sampling_ratio/mean': 1.0141446590423584, 'sampling/importance_sampling_ratio/max': 1.5144426822662354, 'kl': 0.0, 'entropy': 0.10277800261974335, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 3.755320645999973, 'epoch': 0.03125}
22
+ {'loss': 0.0458, 'grad_norm': 4.3092169761657715, 'learning_rate': 5e-07, 'num_tokens': 6857.0, 'completions/mean_length': 19.25, 'completions/min_length': 12.0, 'completions/max_length': 24.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 19.25, 'completions/min_terminated_length': 12.0, 'completions/max_terminated_length': 24.0, 'rewards/countdown_reward/mean': 0.03125, 'rewards/countdown_reward/std': 0.1767766922712326, 'rewards/format_reward/mean': 0.10000000149011612, 'rewards/format_reward/std': 0.0, 'rewards/valid_numbers_reward/mean': 0.16249999403953552, 'rewards/valid_numbers_reward/std': 0.0793115496635437, 'rewards/proximity_reward/mean': 0.07604166865348816, 'rewards/proximity_reward/std': 0.044184666126966476, 'reward': 0.3697916567325592, 'reward_std': 0.2362125664949417, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.01406970713287592, 'sampling/sampling_logp_difference/max': 0.6686638593673706, 'sampling/importance_sampling_ratio/min': 0.4218193590641022, 'sampling/importance_sampling_ratio/mean': 1.0178477764129639, 'sampling/importance_sampling_ratio/max': 1.8781459331512451, 'kl': 0.0, 'entropy': 0.06965943425893784, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 0.7183536330001061, 'epoch': 0.0625}
23
+ {'loss': -0.0573, 'grad_norm': 2.5476601123809814, 'learning_rate': 1e-06, 'num_tokens': 10398.0, 'completions/mean_length': 22.15625, 'completions/min_length': 18.0, 'completions/max_length': 23.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 22.15625, 'completions/min_terminated_length': 18.0, 'completions/max_terminated_length': 23.0, 'rewards/countdown_reward/mean': 0.21875, 'rewards/countdown_reward/std': 0.420013427734375, 'rewards/format_reward/mean': 0.05000000074505806, 'rewards/format_reward/std': 0.05080005154013634, 'rewards/valid_numbers_reward/mean': 0.08750000596046448, 'rewards/valid_numbers_reward/std': 0.10080322623252869, 'rewards/proximity_reward/mean': 0.04661458730697632, 'rewards/proximity_reward/std': 0.08285506069660187, 'reward': 0.40286460518836975, 'reward_std': 0.6027826070785522, 'frac_reward_zero_std': 0.5, 'sampling/sampling_logp_difference/mean': 0.014005242846906185, 'sampling/sampling_logp_difference/max': 1.7997989654541016, 'sampling/importance_sampling_ratio/min': 0.15099729597568512, 'sampling/importance_sampling_ratio/mean': 1.0347709655761719, 'sampling/importance_sampling_ratio/max': 2.864128351211548, 'kl': 0.001527672982774675, 'entropy': 0.0723511204123497, 'clip_ratio/low_mean': 0.007052185945212841, 'clip_ratio/low_min': 0.007052185945212841, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.007052185945212841, 'step_time': 0.7124838239999463, 'epoch': 0.09375}
24
+ {'loss': 0.009, 'grad_norm': 2.3399267196655273, 'learning_rate': 9.982922465033348e-07, 'num_tokens': 13925.0, 'completions/mean_length': 22.21875, 'completions/min_length': 18.0, 'completions/max_length': 26.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 22.21875, 'completions/min_terminated_length': 18.0, 'completions/max_terminated_length': 26.0, 'rewards/countdown_reward/mean': 0.71875, 'rewards/countdown_reward/std': 0.45680341124534607, 'rewards/format_reward/mean': 0.10000000149011612, 'rewards/format_reward/std': 0.0, 'rewards/valid_numbers_reward/mean': 0.1875, 'rewards/valid_numbers_reward/std': 0.049186933785676956, 'rewards/proximity_reward/mean': 0.15000000596046448, 'rewards/proximity_reward/std': 0.08146130293607712, 'reward': 1.15625, 'reward_std': 0.5606581568717957, 'frac_reward_zero_std': 0.5, 'sampling/sampling_logp_difference/mean': 0.0076187350787222385, 'sampling/sampling_logp_difference/max': 0.7177844047546387, 'sampling/importance_sampling_ratio/min': 0.6298430562019348, 'sampling/importance_sampling_ratio/mean': 1.050743579864502, 'sampling/importance_sampling_ratio/max': 2.0516271591186523, 'kl': 0.001772764720954001, 'entropy': 0.05113045871257782, 'clip_ratio/low_mean': 0.002812939463183284, 'clip_ratio/low_min': 0.002812939463183284, 'clip_ratio/high_mean': 0.001406469731591642, 'clip_ratio/high_max': 0.001406469731591642, 'clip_ratio/region_mean': 0.004219409078359604, 'step_time': 0.7690993129999697, 'epoch': 0.125}
25
+ {'loss': -0.0315, 'grad_norm': 3.5514087677001953, 'learning_rate': 9.931806517013612e-07, 'num_tokens': 17452.0, 'completions/mean_length': 21.21875, 'completions/min_length': 18.0, 'completions/max_length': 24.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 21.21875, 'completions/min_terminated_length': 18.0, 'completions/max_terminated_length': 24.0, 'rewards/countdown_reward/mean': 0.53125, 'rewards/countdown_reward/std': 0.507007360458374, 'rewards/format_reward/mean': 0.08437500149011612, 'rewards/format_reward/std': 0.03689020499587059, 'rewards/valid_numbers_reward/mean': 0.16875000298023224, 'rewards/valid_numbers_reward/std': 0.07378040999174118, 'rewards/proximity_reward/mean': 0.12217411398887634, 'rewards/proximity_reward/std': 0.08884944021701813, 'reward': 0.9065490961074829, 'reward_std': 0.6525263786315918, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.023038361221551895, 'sampling/sampling_logp_difference/max': 0.7486381530761719, 'sampling/importance_sampling_ratio/min': 0.0, 'sampling/importance_sampling_ratio/mean': 0.9587581157684326, 'sampling/importance_sampling_ratio/max': 1.7389065027236938, 'kl': 0.0046511460095644, 'entropy': 0.06443575024604797, 'clip_ratio/low_mean': 0.013254786841571331, 'clip_ratio/low_min': 0.013254786841571331, 'clip_ratio/high_mean': 0.005891016218811274, 'clip_ratio/high_max': 0.005891016218811274, 'clip_ratio/region_mean': 0.019145803526043892, 'step_time': 0.7193690340000103, 'epoch': 0.15625}
26
+ {'loss': 0.0378, 'grad_norm': 2.291996955871582, 'learning_rate': 9.847001329696652e-07, 'num_tokens': 20879.0, 'completions/mean_length': 19.59375, 'completions/min_length': 18.0, 'completions/max_length': 22.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 19.59375, 'completions/min_terminated_length': 18.0, 'completions/max_terminated_length': 22.0, 'rewards/countdown_reward/mean': 0.0, 'rewards/countdown_reward/std': 0.0, 'rewards/format_reward/mean': 0.10000000149011612, 'rewards/format_reward/std': 0.0, 'rewards/valid_numbers_reward/mean': 0.20000000298023224, 'rewards/valid_numbers_reward/std': 0.0, 'rewards/proximity_reward/mean': 0.054739583283662796, 'rewards/proximity_reward/std': 0.04368418827652931, 'reward': 0.35473960638046265, 'reward_std': 0.043684203177690506, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.01503602135926485, 'sampling/sampling_logp_difference/max': 0.6983115673065186, 'sampling/importance_sampling_ratio/min': 0.49442461133003235, 'sampling/importance_sampling_ratio/mean': 1.021022081375122, 'sampling/importance_sampling_ratio/max': 1.8775709867477417, 'kl': 0.0014773447765037417, 'entropy': 0.07872910797595978, 'clip_ratio/low_mean': 0.003189792623743415, 'clip_ratio/low_min': 0.003189792623743415, 'clip_ratio/high_mean': 0.0047846888191998005, 'clip_ratio/high_max': 0.0047846888191998005, 'clip_ratio/region_mean': 0.007974481210112572, 'step_time': 0.6970211140001084, 'epoch': 0.1875}
27
+ {'loss': 0.0806, 'grad_norm': 3.5400500297546387, 'learning_rate': 9.729086208503173e-07, 'num_tokens': 24397.0, 'completions/mean_length': 21.4375, 'completions/min_length': 18.0, 'completions/max_length': 25.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 21.4375, 'completions/min_terminated_length': 18.0, 'completions/max_terminated_length': 25.0, 'rewards/countdown_reward/mean': 0.03125, 'rewards/countdown_reward/std': 0.1767766922712326, 'rewards/format_reward/mean': 0.09375, 'rewards/format_reward/std': 0.024593466892838478, 'rewards/valid_numbers_reward/mean': 0.1875, 'rewards/valid_numbers_reward/std': 0.049186933785676956, 'rewards/proximity_reward/mean': 0.026139114052057266, 'rewards/proximity_reward/std': 0.03372207283973694, 'reward': 0.33863911032676697, 'reward_std': 0.22642673552036285, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.00891045667231083, 'sampling/sampling_logp_difference/max': 0.3650592565536499, 'sampling/importance_sampling_ratio/min': 0.548480212688446, 'sampling/importance_sampling_ratio/mean': 1.071790099143982, 'sampling/importance_sampling_ratio/max': 2.0098817348480225, 'kl': 0.0005612155073322356, 'entropy': 0.05858924612402916, 'clip_ratio/low_mean': 0.0029154520016163588, 'clip_ratio/low_min': 0.0029154520016163588, 'clip_ratio/high_mean': 0.004373177886009216, 'clip_ratio/high_max': 0.004373177886009216, 'clip_ratio/region_mean': 0.007288629654794931, 'step_time': 0.7219514629998685, 'epoch': 0.21875}
28
+ {'loss': 0.1165, 'grad_norm': 3.065430164337158, 'learning_rate': 9.578866633275286e-07, 'num_tokens': 27842.0, 'completions/mean_length': 19.65625, 'completions/min_length': 16.0, 'completions/max_length': 23.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 19.65625, 'completions/min_terminated_length': 16.0, 'completions/max_terminated_length': 23.0, 'rewards/countdown_reward/mean': 0.34375, 'rewards/countdown_reward/std': 0.4825586974620819, 'rewards/format_reward/mean': 0.09375, 'rewards/format_reward/std': 0.024593466892838478, 'rewards/valid_numbers_reward/mean': 0.17500001192092896, 'rewards/valid_numbers_reward/std': 0.06720215082168579, 'rewards/proximity_reward/mean': 0.10104167461395264, 'rewards/proximity_reward/std': 0.07623975723981857, 'reward': 0.7135416269302368, 'reward_std': 0.5873492956161499, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.01804327219724655, 'sampling/sampling_logp_difference/max': 0.6249973773956299, 'sampling/importance_sampling_ratio/min': 0.6445150971412659, 'sampling/importance_sampling_ratio/mean': 1.0156548023223877, 'sampling/importance_sampling_ratio/max': 2.68859601020813, 'kl': 0.0026622372679412365, 'entropy': 0.08564577996730804, 'clip_ratio/low_mean': 0.007949125953018665, 'clip_ratio/low_min': 0.007949125953018665, 'clip_ratio/high_mean': 0.007949125953018665, 'clip_ratio/high_max': 0.007949125953018665, 'clip_ratio/region_mean': 0.01589825190603733, 'step_time': 0.7035421840000708, 'epoch': 0.25}
29
+ {'loss': -0.0592, 'grad_norm': 3.6667890548706055, 'learning_rate': 9.397368756032444e-07, 'num_tokens': 31347.0, 'completions/mean_length': 20.53125, 'completions/min_length': 18.0, 'completions/max_length': 23.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 20.53125, 'completions/min_terminated_length': 18.0, 'completions/max_terminated_length': 23.0, 'rewards/countdown_reward/mean': 0.5, 'rewards/countdown_reward/std': 0.5080004930496216, 'rewards/format_reward/mean': 0.05312500149011612, 'rewards/format_reward/std': 0.05070073530077934, 'rewards/valid_numbers_reward/mean': 0.10625000298023224, 'rewards/valid_numbers_reward/std': 0.10140147060155869, 'rewards/proximity_reward/mean': 0.10078125447034836, 'rewards/proximity_reward/std': 0.10090005397796631, 'reward': 0.7601562738418579, 'reward_std': 0.7538037896156311, 'frac_reward_zero_std': 0.5, 'sampling/sampling_logp_difference/mean': 0.0033016959205269814, 'sampling/sampling_logp_difference/max': 0.49289751052856445, 'sampling/importance_sampling_ratio/min': 0.8354066014289856, 'sampling/importance_sampling_ratio/mean': 1.010882019996643, 'sampling/importance_sampling_ratio/max': 1.7604669332504272, 'kl': 0.0012082780012860894, 'entropy': 0.026930633932352066, 'clip_ratio/low_mean': 0.0015220700297504663, 'clip_ratio/low_min': 0.0015220700297504663, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0015220700297504663, 'step_time': 0.7023271729999578, 'epoch': 0.28125}
30
+ {'loss': 0.0201, 'grad_norm': 4.752869606018066, 'learning_rate': 9.185832391312642e-07, 'num_tokens': 34845.0, 'completions/mean_length': 20.8125, 'completions/min_length': 18.0, 'completions/max_length': 23.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 20.8125, 'completions/min_terminated_length': 18.0, 'completions/max_terminated_length': 23.0, 'rewards/countdown_reward/mean': 0.625, 'rewards/countdown_reward/std': 0.49186936020851135, 'rewards/format_reward/mean': 0.09375, 'rewards/format_reward/std': 0.024593466892838478, 'rewards/valid_numbers_reward/mean': 0.1875, 'rewards/valid_numbers_reward/std': 0.049186933785676956, 'rewards/proximity_reward/mean': 0.13895833492279053, 'rewards/proximity_reward/std': 0.08098160475492477, 'reward': 1.0452083349227905, 'reward_std': 0.6018912196159363, 'frac_reward_zero_std': 0.5, 'sampling/sampling_logp_difference/mean': 0.014039169996976852, 'sampling/sampling_logp_difference/max': 0.6249973773956299, 'sampling/importance_sampling_ratio/min': 0.4273960292339325, 'sampling/importance_sampling_ratio/mean': 1.032275915145874, 'sampling/importance_sampling_ratio/max': 2.3014140129089355, 'kl': 0.0016276317182928324, 'entropy': 0.07305138558149338, 'clip_ratio/low_mean': 0.006006006151437759, 'clip_ratio/low_min': 0.006006006151437759, 'clip_ratio/high_mean': 0.0015015015378594398, 'clip_ratio/high_max': 0.0015015015378594398, 'clip_ratio/region_mean': 0.007507507689297199, 'step_time': 0.7014926839999589, 'epoch': 0.3125}
31
+ {'loss': -0.1784, 'grad_norm': 6.029069423675537, 'learning_rate': 8.945702546981968e-07, 'num_tokens': 38261.0, 'completions/mean_length': 18.25, 'completions/min_length': 12.0, 'completions/max_length': 24.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 18.25, 'completions/min_terminated_length': 12.0, 'completions/max_terminated_length': 24.0, 'rewards/countdown_reward/mean': 0.0, 'rewards/countdown_reward/std': 0.0, 'rewards/format_reward/mean': 0.10000000149011612, 'rewards/format_reward/std': 0.0, 'rewards/valid_numbers_reward/mean': 0.15625, 'rewards/valid_numbers_reward/std': 0.08400268852710724, 'rewards/proximity_reward/mean': 0.04296875, 'rewards/proximity_reward/std': 0.040775928646326065, 'reward': 0.2992187738418579, 'reward_std': 0.11225052177906036, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.014431153424084187, 'sampling/sampling_logp_difference/max': 0.5645503997802734, 'sampling/importance_sampling_ratio/min': 0.5598352551460266, 'sampling/importance_sampling_ratio/mean': 1.0049711465835571, 'sampling/importance_sampling_ratio/max': 2.108710527420044, 'kl': 0.0014566656900569797, 'entropy': 0.08794382214546204, 'clip_ratio/low_mean': 0.0017123287543654442, 'clip_ratio/low_min': 0.0017123287543654442, 'clip_ratio/high_mean': 0.006849315017461777, 'clip_ratio/high_max': 0.006849315017461777, 'clip_ratio/region_mean': 0.008561643771827221, 'step_time': 0.7050704440000573, 'epoch': 0.34375}
32
+ {'loss': 0.1242, 'grad_norm': 3.043853998184204, 'learning_rate': 8.678619553365658e-07, 'num_tokens': 41675.0, 'completions/mean_length': 18.1875, 'completions/min_length': 12.0, 'completions/max_length': 25.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 18.1875, 'completions/min_terminated_length': 12.0, 'completions/max_terminated_length': 25.0, 'rewards/countdown_reward/mean': 0.5, 'rewards/countdown_reward/std': 0.5080004930496216, 'rewards/format_reward/mean': 0.09687499701976776, 'rewards/format_reward/std': 0.01767767034471035, 'rewards/valid_numbers_reward/mean': 0.10000000149011612, 'rewards/valid_numbers_reward/std': 0.10160010308027267, 'rewards/proximity_reward/mean': 0.10000000149011612, 'rewards/proximity_reward/std': 0.10160010308027267, 'reward': 0.796875, 'reward_std': 0.7145873308181763, 'frac_reward_zero_std': 0.5, 'sampling/sampling_logp_difference/mean': 0.007687629200518131, 'sampling/sampling_logp_difference/max': 0.5967673063278198, 'sampling/importance_sampling_ratio/min': 0.6460556387901306, 'sampling/importance_sampling_ratio/mean': 0.9738063812255859, 'sampling/importance_sampling_ratio/max': 1.5643600225448608, 'kl': 0.0004984450642950833, 'entropy': 0.05066652595996857, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0017182130832225084, 'clip_ratio/high_max': 0.0017182130832225084, 'clip_ratio/region_mean': 0.0017182130832225084, 'step_time': 0.7119393329999184, 'epoch': 0.375}
33
+ {'loss': -0.0156, 'grad_norm': 2.8895351886749268, 'learning_rate': 8.386407858128706e-07, 'num_tokens': 45144.0, 'completions/mean_length': 20.40625, 'completions/min_length': 18.0, 'completions/max_length': 23.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 20.40625, 'completions/min_terminated_length': 18.0, 'completions/max_terminated_length': 23.0, 'rewards/countdown_reward/mean': 0.5, 'rewards/countdown_reward/std': 0.5080004930496216, 'rewards/format_reward/mean': 0.10000000149011612, 'rewards/format_reward/std': 0.0, 'rewards/valid_numbers_reward/mean': 0.20000000298023224, 'rewards/valid_numbers_reward/std': 0.0, 'rewards/proximity_reward/mean': 0.10451087355613708, 'rewards/proximity_reward/std': 0.09701789915561676, 'reward': 0.9045108556747437, 'reward_std': 0.6050176620483398, 'frac_reward_zero_std': 0.5, 'sampling/sampling_logp_difference/mean': 0.0066388980485498905, 'sampling/sampling_logp_difference/max': 0.3109920620918274, 'sampling/importance_sampling_ratio/min': 0.7341673970222473, 'sampling/importance_sampling_ratio/mean': 0.9615199565887451, 'sampling/importance_sampling_ratio/max': 1.2092349529266357, 'kl': 0.000671109592076391, 'entropy': 0.050037682056427, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0015313936164602637, 'clip_ratio/high_max': 0.0015313936164602637, 'clip_ratio/region_mean': 0.0015313936164602637, 'step_time': 0.6962026339999738, 'epoch': 0.40625}
34
+ {'loss': 0.0331, 'grad_norm': 1.8920438289642334, 'learning_rate': 8.071063563448339e-07, 'num_tokens': 49016.0, 'completions/mean_length': 33.0, 'completions/min_length': 18.0, 'completions/max_length': 307.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 33.0, 'completions/min_terminated_length': 18.0, 'completions/max_terminated_length': 307.0, 'rewards/countdown_reward/mean': 0.0, 'rewards/countdown_reward/std': 0.0, 'rewards/format_reward/mean': 0.012500000186264515, 'rewards/format_reward/std': 0.033601075410842896, 'rewards/valid_numbers_reward/mean': 0.02500000037252903, 'rewards/valid_numbers_reward/std': 0.06720215082168579, 'rewards/proximity_reward/mean': 0.004999999888241291, 'rewards/proximity_reward/std': 0.013440429233014584, 'reward': 0.042500000447034836, 'reward_std': 0.11424365639686584, 'frac_reward_zero_std': 0.5, 'sampling/sampling_logp_difference/mean': 0.007544555701315403, 'sampling/sampling_logp_difference/max': 0.3881516456604004, 'sampling/importance_sampling_ratio/min': 0.7685970067977905, 'sampling/importance_sampling_ratio/mean': 1.0223689079284668, 'sampling/importance_sampling_ratio/max': 1.4428951740264893, 'kl': 0.0007539401995018125, 'entropy': 0.10343185812234879, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 3.272125610999865, 'epoch': 0.4375}
35
+ {'loss': 0.1982, 'grad_norm': 4.963200092315674, 'learning_rate': 7.734740790612136e-07, 'num_tokens': 52421.0, 'completions/mean_length': 18.90625, 'completions/min_length': 18.0, 'completions/max_length': 22.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 18.90625, 'completions/min_terminated_length': 18.0, 'completions/max_terminated_length': 22.0, 'rewards/countdown_reward/mean': 0.0, 'rewards/countdown_reward/std': 0.0, 'rewards/format_reward/mean': 0.08124999701976776, 'rewards/format_reward/std': 0.03965577483177185, 'rewards/valid_numbers_reward/mean': 0.16249999403953552, 'rewards/valid_numbers_reward/std': 0.0793115496635437, 'rewards/proximity_reward/mean': 0.03890354186296463, 'rewards/proximity_reward/std': 0.03131883218884468, 'reward': 0.2826535403728485, 'reward_std': 0.14018532633781433, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.01146075502038002, 'sampling/sampling_logp_difference/max': 0.6627943515777588, 'sampling/importance_sampling_ratio/min': 0.49324995279312134, 'sampling/importance_sampling_ratio/mean': 0.9984955787658691, 'sampling/importance_sampling_ratio/max': 1.7686353921890259, 'kl': 0.00044077940401621163, 'entropy': 0.05800556764006615, 'clip_ratio/low_mean': 0.0016528925625607371, 'clip_ratio/low_min': 0.0016528925625607371, 'clip_ratio/high_mean': 0.00826446246355772, 'clip_ratio/high_max': 0.00826446246355772, 'clip_ratio/region_mean': 0.00991735514253378, 'step_time': 0.6855096929998581, 'epoch': 0.46875}
36
+ {'loss': -0.0484, 'grad_norm': 2.715698480606079, 'learning_rate': 7.379736965185368e-07, 'num_tokens': 55881.0, 'completions/mean_length': 21.125, 'completions/min_length': 18.0, 'completions/max_length': 24.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 21.125, 'completions/min_terminated_length': 18.0, 'completions/max_terminated_length': 24.0, 'rewards/countdown_reward/mean': 0.25, 'rewards/countdown_reward/std': 0.4399413466453552, 'rewards/format_reward/mean': 0.09687499701976776, 'rewards/format_reward/std': 0.01767767034471035, 'rewards/valid_numbers_reward/mean': 0.10000000149011612, 'rewards/valid_numbers_reward/std': 0.10160010308027267, 'rewards/proximity_reward/mean': 0.0627967119216919, 'rewards/proximity_reward/std': 0.0837658941745758, 'reward': 0.509671688079834, 'reward_std': 0.5909783244132996, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.015001348219811916, 'sampling/sampling_logp_difference/max': 0.7091498374938965, 'sampling/importance_sampling_ratio/min': 0.36538299918174744, 'sampling/importance_sampling_ratio/mean': 0.9959367513656616, 'sampling/importance_sampling_ratio/max': 1.694196105003357, 'kl': 0.0013793795369565487, 'entropy': 0.10765296965837479, 'clip_ratio/low_mean': 0.002958579920232296, 'clip_ratio/low_min': 0.002958579920232296, 'clip_ratio/high_mean': 0.002958579920232296, 'clip_ratio/high_max': 0.002958579920232296, 'clip_ratio/region_mean': 0.005917159840464592, 'step_time': 0.6983848740001122, 'epoch': 0.5}
37
+ {'loss': 0.0272, 'grad_norm': 1.209149718284607, 'learning_rate': 7.008477123264847e-07, 'num_tokens': 59364.0, 'completions/mean_length': 21.34375, 'completions/min_length': 18.0, 'completions/max_length': 29.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 21.34375, 'completions/min_terminated_length': 18.0, 'completions/max_terminated_length': 29.0, 'rewards/countdown_reward/mean': 0.0, 'rewards/countdown_reward/std': 0.0, 'rewards/format_reward/mean': 0.05000000074505806, 'rewards/format_reward/std': 0.05080005154013634, 'rewards/valid_numbers_reward/mean': 0.10000000149011612, 'rewards/valid_numbers_reward/std': 0.10160010308027267, 'rewards/proximity_reward/mean': 0.02812499925494194, 'rewards/proximity_reward/std': 0.03094610385596752, 'reward': 0.17812499403953552, 'reward_std': 0.18136467039585114, 'frac_reward_zero_std': 0.5, 'sampling/sampling_logp_difference/mean': 0.004375044722110033, 'sampling/sampling_logp_difference/max': 0.4909186363220215, 'sampling/importance_sampling_ratio/min': 0.5967592597007751, 'sampling/importance_sampling_ratio/mean': 0.9605278968811035, 'sampling/importance_sampling_ratio/max': 1.3050473928451538, 'kl': 0.00023233593674376607, 'entropy': 0.03696838766336441, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0014641288435086608, 'clip_ratio/high_max': 0.0014641288435086608, 'clip_ratio/region_mean': 0.0014641288435086608, 'step_time': 0.7314563830000225, 'epoch': 0.53125}
38
+ {'loss': 0.0942, 'grad_norm': 2.5989675521850586, 'learning_rate': 6.623497346023417e-07, 'num_tokens': 62793.0, 'completions/mean_length': 20.15625, 'completions/min_length': 16.0, 'completions/max_length': 26.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 20.15625, 'completions/min_terminated_length': 16.0, 'completions/max_terminated_length': 26.0, 'rewards/countdown_reward/mean': 0.0, 'rewards/countdown_reward/std': 0.0, 'rewards/format_reward/mean': 0.07187500596046448, 'rewards/format_reward/std': 0.04568034037947655, 'rewards/valid_numbers_reward/mean': 0.11249999701976776, 'rewards/valid_numbers_reward/std': 0.10080322623252869, 'rewards/proximity_reward/mean': 0.03396707400679588, 'rewards/proximity_reward/std': 0.050391241908073425, 'reward': 0.21834206581115723, 'reward_std': 0.17154528200626373, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.014694616198539734, 'sampling/sampling_logp_difference/max': 0.9361472725868225, 'sampling/importance_sampling_ratio/min': 0.5324814915657043, 'sampling/importance_sampling_ratio/mean': 1.020961046218872, 'sampling/importance_sampling_ratio/max': 1.700570821762085, 'kl': 0.002194973174482584, 'entropy': 0.09666672348976135, 'clip_ratio/low_mean': 0.010852713137865067, 'clip_ratio/low_min': 0.010852713137865067, 'clip_ratio/high_mean': 0.006201550364494324, 'clip_ratio/high_max': 0.006201550364494324, 'clip_ratio/region_mean': 0.01705426350235939, 'step_time': 0.7106611730000623, 'epoch': 0.5625}
39
+ {'loss': 0.1011, 'grad_norm': 3.349228858947754, 'learning_rate': 6.227427435703995e-07, 'num_tokens': 66155.0, 'completions/mean_length': 17.0625, 'completions/min_length': 12.0, 'completions/max_length': 28.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 17.0625, 'completions/min_terminated_length': 12.0, 'completions/max_terminated_length': 28.0, 'rewards/countdown_reward/mean': 0.3125, 'rewards/countdown_reward/std': 0.4709290862083435, 'rewards/format_reward/mean': 0.09062500298023224, 'rewards/format_reward/std': 0.029614459723234177, 'rewards/valid_numbers_reward/mean': 0.06875000149011612, 'rewards/valid_numbers_reward/std': 0.09651173651218414, 'rewards/proximity_reward/mean': 0.06302083283662796, 'rewards/proximity_reward/std': 0.09387461096048355, 'reward': 0.5348958373069763, 'reward_std': 0.6629514694213867, 'frac_reward_zero_std': 0.5, 'sampling/sampling_logp_difference/mean': 0.0075608654879033566, 'sampling/sampling_logp_difference/max': 0.730109691619873, 'sampling/importance_sampling_ratio/min': 0.7518770694732666, 'sampling/importance_sampling_ratio/mean': 1.0364713668823242, 'sampling/importance_sampling_ratio/max': 2.0383474826812744, 'kl': 0.002250331686809659, 'entropy': 0.06416559964418411, 'clip_ratio/low_mean': 0.005494505632668734, 'clip_ratio/low_min': 0.005494505632668734, 'clip_ratio/high_mean': 0.005494505632668734, 'clip_ratio/high_max': 0.005494505632668734, 'clip_ratio/region_mean': 0.010989011265337467, 'step_time': 0.7281739340000968, 'epoch': 0.59375}
40
+ {'loss': -0.1065, 'grad_norm': 3.5681610107421875, 'learning_rate': 5.82297295140367e-07, 'num_tokens': 69662.0, 'completions/mean_length': 21.09375, 'completions/min_length': 19.0, 'completions/max_length': 27.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 21.09375, 'completions/min_terminated_length': 19.0, 'completions/max_terminated_length': 27.0, 'rewards/countdown_reward/mean': 0.1875, 'rewards/countdown_reward/std': 0.3965577781200409, 'rewards/format_reward/mean': 0.08750000596046448, 'rewards/format_reward/std': 0.033601075410842896, 'rewards/valid_numbers_reward/mean': 0.09375, 'rewards/valid_numbers_reward/std': 0.10140147060155869, 'rewards/proximity_reward/mean': 0.038453392684459686, 'rewards/proximity_reward/std': 0.07886006683111191, 'reward': 0.40720340609550476, 'reward_std': 0.5433569550514221, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.011155059561133385, 'sampling/sampling_logp_difference/max': 0.7159504890441895, 'sampling/importance_sampling_ratio/min': 0.5663295388221741, 'sampling/importance_sampling_ratio/mean': 1.0255656242370605, 'sampling/importance_sampling_ratio/max': 1.3683027029037476, 'kl': 0.0022981727961450815, 'entropy': 0.08511413633823395, 'clip_ratio/low_mean': 0.00296296295709908, 'clip_ratio/low_min': 0.00296296295709908, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.00296296295709908, 'step_time': 0.7250004739998985, 'epoch': 0.625}
41
+ {'loss': 0.0344, 'grad_norm': 1.683887243270874, 'learning_rate': 5.412896727361662e-07, 'num_tokens': 73220.0, 'completions/mean_length': 22.1875, 'completions/min_length': 19.0, 'completions/max_length': 23.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 22.1875, 'completions/min_terminated_length': 19.0, 'completions/max_terminated_length': 23.0, 'rewards/countdown_reward/mean': 0.5, 'rewards/countdown_reward/std': 0.5080004930496216, 'rewards/format_reward/mean': 0.10000000149011612, 'rewards/format_reward/std': 0.0, 'rewards/valid_numbers_reward/mean': 0.20000000298023224, 'rewards/valid_numbers_reward/std': 0.0, 'rewards/proximity_reward/mean': 0.12166666984558105, 'rewards/proximity_reward/std': 0.07983854413032532, 'reward': 0.92166668176651, 'reward_std': 0.5876213908195496, 'frac_reward_zero_std': 0.5, 'sampling/sampling_logp_difference/mean': 0.005603067576885223, 'sampling/sampling_logp_difference/max': 0.48111605644226074, 'sampling/importance_sampling_ratio/min': 0.7776699066162109, 'sampling/importance_sampling_ratio/mean': 1.0168145895004272, 'sampling/importance_sampling_ratio/max': 1.4217238426208496, 'kl': 0.0009874801617115736, 'entropy': 0.028706880286335945, 'clip_ratio/low_mean': 0.002816901309415698, 'clip_ratio/low_min': 0.002816901309415698, 'clip_ratio/high_mean': 0.002816901309415698, 'clip_ratio/high_max': 0.002816901309415698, 'clip_ratio/region_mean': 0.005633802618831396, 'step_time': 0.6987281040001108, 'epoch': 0.65625}
42
+ {'loss': -0.0805, 'grad_norm': 2.383918285369873, 'learning_rate': 5e-07, 'num_tokens': 76583.0, 'completions/mean_length': 17.59375, 'completions/min_length': 12.0, 'completions/max_length': 23.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 17.59375, 'completions/min_terminated_length': 12.0, 'completions/max_terminated_length': 23.0, 'rewards/countdown_reward/mean': 0.0, 'rewards/countdown_reward/std': 0.0, 'rewards/format_reward/mean': 0.05000000074505806, 'rewards/format_reward/std': 0.05080005154013634, 'rewards/valid_numbers_reward/mean': 0.01875000074505806, 'rewards/valid_numbers_reward/std': 0.05922891944646835, 'rewards/proximity_reward/mean': 0.0026785715017467737, 'rewards/proximity_reward/std': 0.008461273275315762, 'reward': 0.0714285746216774, 'reward_std': 0.09682246297597885, 'frac_reward_zero_std': 0.5, 'sampling/sampling_logp_difference/mean': 0.0034464546479284763, 'sampling/sampling_logp_difference/max': 0.3867391347885132, 'sampling/importance_sampling_ratio/min': 0.7479129433631897, 'sampling/importance_sampling_ratio/mean': 0.9816645383834839, 'sampling/importance_sampling_ratio/max': 1.3836370706558228, 'kl': 0.0002953831572085619, 'entropy': 0.03434375301003456, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0017761989729478955, 'clip_ratio/high_max': 0.0017761989729478955, 'clip_ratio/region_mean': 0.0017761989729478955, 'step_time': 0.6868111840001347, 'epoch': 0.6875}
43
+ {'loss': 0.0335, 'grad_norm': 1.8223655223846436, 'learning_rate': 4.5871032726383385e-07, 'num_tokens': 80047.0, 'completions/mean_length': 20.25, 'completions/min_length': 18.0, 'completions/max_length': 23.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 20.25, 'completions/min_terminated_length': 18.0, 'completions/max_terminated_length': 23.0, 'rewards/countdown_reward/mean': 0.03125, 'rewards/countdown_reward/std': 0.1767766922712326, 'rewards/format_reward/mean': 0.10000000149011612, 'rewards/format_reward/std': 0.0, 'rewards/valid_numbers_reward/mean': 0.1875, 'rewards/valid_numbers_reward/std': 0.049186933785676956, 'rewards/proximity_reward/mean': 0.0259108766913414, 'rewards/proximity_reward/std': 0.04473352059721947, 'reward': 0.3446608781814575, 'reward_std': 0.21994799375534058, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.010006303898990154, 'sampling/sampling_logp_difference/max': 0.6734256744384766, 'sampling/importance_sampling_ratio/min': 0.568607747554779, 'sampling/importance_sampling_ratio/mean': 1.0024545192718506, 'sampling/importance_sampling_ratio/max': 2.2461788654327393, 'kl': 0.0014518073294311762, 'entropy': 0.07266536355018616, 'clip_ratio/low_mean': 0.009259259328246117, 'clip_ratio/low_min': 0.009259259328246117, 'clip_ratio/high_mean': 0.004629629664123058, 'clip_ratio/high_max': 0.004629629664123058, 'clip_ratio/region_mean': 0.013888888992369175, 'step_time': 0.6992476539999188, 'epoch': 0.71875}
44
+ {'loss': -0.114, 'grad_norm': 5.045168399810791, 'learning_rate': 4.1770270485963294e-07, 'num_tokens': 83353.0, 'completions/mean_length': 16.3125, 'completions/min_length': 12.0, 'completions/max_length': 21.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 16.3125, 'completions/min_terminated_length': 12.0, 'completions/max_terminated_length': 21.0, 'rewards/countdown_reward/mean': 0.0, 'rewards/countdown_reward/std': 0.0, 'rewards/format_reward/mean': 0.05312500149011612, 'rewards/format_reward/std': 0.05070073530077934, 'rewards/valid_numbers_reward/mean': 0.0062500000931322575, 'rewards/valid_numbers_reward/std': 0.0353553406894207, 'rewards/proximity_reward/mean': 0.000480769231216982, 'rewards/proximity_reward/std': 0.0027196414303034544, 'reward': 0.05985577031970024, 'reward_std': 0.06834935396909714, 'frac_reward_zero_std': 0.5, 'sampling/sampling_logp_difference/mean': 0.014905628748238087, 'sampling/sampling_logp_difference/max': 1.1528494358062744, 'sampling/importance_sampling_ratio/min': 0.3555085062980652, 'sampling/importance_sampling_ratio/mean': 0.9059621691703796, 'sampling/importance_sampling_ratio/max': 1.7516393661499023, 'kl': 0.0012164042564108968, 'entropy': 0.042202427983284, 'clip_ratio/low_mean': 0.005747126415371895, 'clip_ratio/low_min': 0.005747126415371895, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.005747126415371895, 'step_time': 0.6691923840000982, 'epoch': 0.75}
45
+ {'loss': -0.046, 'grad_norm': 3.4389476776123047, 'learning_rate': 3.772572564296004e-07, 'num_tokens': 86848.0, 'completions/mean_length': 21.21875, 'completions/min_length': 13.0, 'completions/max_length': 23.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 21.21875, 'completions/min_terminated_length': 13.0, 'completions/max_terminated_length': 23.0, 'rewards/countdown_reward/mean': 0.0625, 'rewards/countdown_reward/std': 0.24593468010425568, 'rewards/format_reward/mean': 0.09062500298023224, 'rewards/format_reward/std': 0.029614457860589027, 'rewards/valid_numbers_reward/mean': 0.17500001192092896, 'rewards/valid_numbers_reward/std': 0.06720215082168579, 'rewards/proximity_reward/mean': 0.057149626314640045, 'rewards/proximity_reward/std': 0.044902537018060684, 'reward': 0.38527464866638184, 'reward_std': 0.3133750557899475, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.01222001202404499, 'sampling/sampling_logp_difference/max': 0.8772695064544678, 'sampling/importance_sampling_ratio/min': 0.5971362590789795, 'sampling/importance_sampling_ratio/mean': 1.023521900177002, 'sampling/importance_sampling_ratio/max': 2.4219090938568115, 'kl': 0.0022281454876065254, 'entropy': 0.07401835173368454, 'clip_ratio/low_mean': 0.0014727540547028184, 'clip_ratio/low_min': 0.0014727540547028184, 'clip_ratio/high_mean': 0.011782032437622547, 'clip_ratio/high_max': 0.011782032437622547, 'clip_ratio/region_mean': 0.013254786841571331, 'step_time': 0.6923223940000298, 'epoch': 0.78125}
46
+ {'loss': -0.0309, 'grad_norm': 1.6091957092285156, 'learning_rate': 3.3765026539765827e-07, 'num_tokens': 90326.0, 'completions/mean_length': 19.6875, 'completions/min_length': 18.0, 'completions/max_length': 25.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 19.6875, 'completions/min_terminated_length': 18.0, 'completions/max_terminated_length': 25.0, 'rewards/countdown_reward/mean': 0.59375, 'rewards/countdown_reward/std': 0.49899089336395264, 'rewards/format_reward/mean': 0.10000000149011612, 'rewards/format_reward/std': 0.0, 'rewards/valid_numbers_reward/mean': 0.20000000298023224, 'rewards/valid_numbers_reward/std': 0.0, 'rewards/proximity_reward/mean': 0.1383928656578064, 'rewards/proximity_reward/std': 0.07889021933078766, 'reward': 1.0321428775787354, 'reward_std': 0.5750947594642639, 'frac_reward_zero_std': 0.5, 'sampling/sampling_logp_difference/mean': 0.007631146814674139, 'sampling/sampling_logp_difference/max': 0.48116254806518555, 'sampling/importance_sampling_ratio/min': 0.6717771887779236, 'sampling/importance_sampling_ratio/mean': 1.010103702545166, 'sampling/importance_sampling_ratio/max': 1.5968704223632812, 'kl': 0.0010196284856647253, 'entropy': 0.05027592182159424, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0031746032182127237, 'clip_ratio/high_max': 0.0031746032182127237, 'clip_ratio/region_mean': 0.0031746032182127237, 'step_time': 0.7108742530001564, 'epoch': 0.8125}
47
+ {'loss': 0.014, 'grad_norm': 1.6168516874313354, 'learning_rate': 2.9915228767351535e-07, 'num_tokens': 93786.0, 'completions/mean_length': 20.125, 'completions/min_length': 18.0, 'completions/max_length': 22.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 20.125, 'completions/min_terminated_length': 18.0, 'completions/max_terminated_length': 22.0, 'rewards/countdown_reward/mean': 0.15625, 'rewards/countdown_reward/std': 0.3689020276069641, 'rewards/format_reward/mean': 0.06562499701976776, 'rewards/format_reward/std': 0.04825586825609207, 'rewards/valid_numbers_reward/mean': 0.13124999403953552, 'rewards/valid_numbers_reward/std': 0.09651173651218414, 'rewards/proximity_reward/mean': 0.06458333134651184, 'rewards/proximity_reward/std': 0.06663305312395096, 'reward': 0.4177083373069763, 'reward_std': 0.5021805167198181, 'frac_reward_zero_std': 0.5, 'sampling/sampling_logp_difference/mean': 0.006294678431004286, 'sampling/sampling_logp_difference/max': 0.4999983310699463, 'sampling/importance_sampling_ratio/min': 0.757157564163208, 'sampling/importance_sampling_ratio/mean': 0.9581866264343262, 'sampling/importance_sampling_ratio/max': 1.058418869972229, 'kl': 0.0006096140714362264, 'entropy': 0.06097259745001793, 'clip_ratio/low_mean': 0.003105590119957924, 'clip_ratio/low_min': 0.003105590119957924, 'clip_ratio/high_mean': 0.001552795059978962, 'clip_ratio/high_max': 0.001552795059978962, 'clip_ratio/region_mean': 0.004658385179936886, 'step_time': 0.6864994640000077, 'epoch': 0.84375}
48
+ {'loss': -0.0799, 'grad_norm': 3.242023468017578, 'learning_rate': 2.620263034814632e-07, 'num_tokens': 97259.0, 'completions/mean_length': 21.03125, 'completions/min_length': 18.0, 'completions/max_length': 22.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 21.03125, 'completions/min_terminated_length': 18.0, 'completions/max_terminated_length': 22.0, 'rewards/countdown_reward/mean': 0.03125, 'rewards/countdown_reward/std': 0.1767766922712326, 'rewards/format_reward/mean': 0.078125, 'rewards/format_reward/std': 0.04200134426355362, 'rewards/valid_numbers_reward/mean': 0.15625, 'rewards/valid_numbers_reward/std': 0.08400268852710724, 'rewards/proximity_reward/mean': 0.04020833596587181, 'rewards/proximity_reward/std': 0.040338512510061264, 'reward': 0.3058333396911621, 'reward_std': 0.2621923089027405, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.004645894281566143, 'sampling/sampling_logp_difference/max': 0.49219369888305664, 'sampling/importance_sampling_ratio/min': 0.7788451910018921, 'sampling/importance_sampling_ratio/mean': 1.0141183137893677, 'sampling/importance_sampling_ratio/max': 1.5722929239273071, 'kl': 0.0011130323400720954, 'entropy': 0.054960351437330246, 'clip_ratio/low_mean': 0.001485884073190391, 'clip_ratio/low_min': 0.001485884073190391, 'clip_ratio/high_mean': 0.001485884073190391, 'clip_ratio/high_max': 0.001485884073190391, 'clip_ratio/region_mean': 0.002971768146380782, 'step_time': 0.691043074000163, 'epoch': 0.875}
49
+ {'loss': -0.1248, 'grad_norm': 2.3634800910949707, 'learning_rate': 2.2652592093878665e-07, 'num_tokens': 100809.0, 'completions/mean_length': 22.4375, 'completions/min_length': 18.0, 'completions/max_length': 25.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 22.4375, 'completions/min_terminated_length': 18.0, 'completions/max_terminated_length': 25.0, 'rewards/countdown_reward/mean': 0.28125, 'rewards/countdown_reward/std': 0.45680341124534607, 'rewards/format_reward/mean': 0.08124999701976776, 'rewards/format_reward/std': 0.03965577483177185, 'rewards/valid_numbers_reward/mean': 0.16249999403953552, 'rewards/valid_numbers_reward/std': 0.0793115496635437, 'rewards/proximity_reward/mean': 0.06205357238650322, 'rewards/proximity_reward/std': 0.08783420920372009, 'reward': 0.5870535373687744, 'reward_std': 0.5920328497886658, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.007978800684213638, 'sampling/sampling_logp_difference/max': 0.9378328323364258, 'sampling/importance_sampling_ratio/min': 0.2862122058868408, 'sampling/importance_sampling_ratio/mean': 1.0282001495361328, 'sampling/importance_sampling_ratio/max': 1.4644712209701538, 'kl': 0.0005595331895165145, 'entropy': 0.03826366737484932, 'clip_ratio/low_mean': 0.002785515272989869, 'clip_ratio/low_min': 0.002785515272989869, 'clip_ratio/high_mean': 0.0013927576364949346, 'clip_ratio/high_max': 0.0013927576364949346, 'clip_ratio/region_mean': 0.0041782730259001255, 'step_time': 0.7093623440000556, 'epoch': 0.90625}
50
+ {'loss': 0.0, 'grad_norm': 0.005207752343267202, 'learning_rate': 1.9289364365516607e-07, 'num_tokens': 104319.0, 'completions/mean_length': 20.6875, 'completions/min_length': 18.0, 'completions/max_length': 24.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 20.6875, 'completions/min_terminated_length': 18.0, 'completions/max_terminated_length': 24.0, 'rewards/countdown_reward/mean': 0.5, 'rewards/countdown_reward/std': 0.5080004930496216, 'rewards/format_reward/mean': 0.10000000149011612, 'rewards/format_reward/std': 0.0, 'rewards/valid_numbers_reward/mean': 0.20000000298023224, 'rewards/valid_numbers_reward/std': 0.0, 'rewards/proximity_reward/mean': 0.12000000476837158, 'rewards/proximity_reward/std': 0.08128008246421814, 'reward': 0.9200000166893005, 'reward_std': 0.5892805457115173, 'frac_reward_zero_std': 1.0, 'sampling/sampling_logp_difference/mean': 0.001935314736329019, 'sampling/sampling_logp_difference/max': 0.2494516372680664, 'sampling/importance_sampling_ratio/min': 0.879340648651123, 'sampling/importance_sampling_ratio/mean': 0.9897561073303223, 'sampling/importance_sampling_ratio/max': 1.3560521602630615, 'kl': 0.00011867708235513419, 'entropy': 0.017748219892382622, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 0.6998922230000062, 'epoch': 0.9375}
51
+ {'loss': 0.0611, 'grad_norm': 2.1904594898223877, 'learning_rate': 1.6135921418712955e-07, 'num_tokens': 107724.0, 'completions/mean_length': 19.40625, 'completions/min_length': 18.0, 'completions/max_length': 23.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 19.40625, 'completions/min_terminated_length': 18.0, 'completions/max_terminated_length': 23.0, 'rewards/countdown_reward/mean': 0.15625, 'rewards/countdown_reward/std': 0.3689020276069641, 'rewards/format_reward/mean': 0.09062500298023224, 'rewards/format_reward/std': 0.029614459723234177, 'rewards/valid_numbers_reward/mean': 0.16875001788139343, 'rewards/valid_numbers_reward/std': 0.07378040999174118, 'rewards/proximity_reward/mean': 0.07500000298023224, 'rewards/proximity_reward/std': 0.061054062098264694, 'reward': 0.4906249940395355, 'reward_std': 0.4572972357273102, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.015460848808288574, 'sampling/sampling_logp_difference/max': 0.8136651515960693, 'sampling/importance_sampling_ratio/min': 0.46403998136520386, 'sampling/importance_sampling_ratio/mean': 0.9687584638595581, 'sampling/importance_sampling_ratio/max': 1.4587351083755493, 'kl': 0.0015718439826741815, 'entropy': 0.08870946615934372, 'clip_ratio/low_mean': 0.004830917809158564, 'clip_ratio/low_min': 0.004830917809158564, 'clip_ratio/high_mean': 0.011272141709923744, 'clip_ratio/high_max': 0.011272141709923744, 'clip_ratio/region_mean': 0.01610305905342102, 'step_time': 0.6862520940001104, 'epoch': 0.96875}
52
+ {'loss': 0.0706, 'grad_norm': 2.8171820640563965, 'learning_rate': 1.321380446634342e-07, 'num_tokens': 111240.0, 'completions/mean_length': 21.875, 'completions/min_length': 18.0, 'completions/max_length': 23.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 21.875, 'completions/min_terminated_length': 18.0, 'completions/max_terminated_length': 23.0, 'rewards/countdown_reward/mean': 0.0, 'rewards/countdown_reward/std': 0.0, 'rewards/format_reward/mean': 0.08749999850988388, 'rewards/format_reward/std': 0.033601075410842896, 'rewards/valid_numbers_reward/mean': 0.13750000298023224, 'rewards/valid_numbers_reward/std': 0.09418581426143646, 'rewards/proximity_reward/mean': 0.0454629622399807, 'rewards/proximity_reward/std': 0.04640965536236763, 'reward': 0.2704629600048065, 'reward_std': 0.15080341696739197, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.009648799896240234, 'sampling/sampling_logp_difference/max': 1.194356918334961, 'sampling/importance_sampling_ratio/min': 0.29941776394844055, 'sampling/importance_sampling_ratio/mean': 0.9770330190658569, 'sampling/importance_sampling_ratio/max': 1.589633822441101, 'kl': 0.0017971686320379376, 'entropy': 0.059439048171043396, 'clip_ratio/low_mean': 0.004285714123398066, 'clip_ratio/low_min': 0.004285714123398066, 'clip_ratio/high_mean': 0.0028571428265422583, 'clip_ratio/high_max': 0.0028571428265422583, 'clip_ratio/region_mean': 0.0071428571827709675, 'step_time': 0.6916781440002069, 'epoch': 1.0}
53
+ {'loss': 0.22, 'grad_norm': 5.006729602813721, 'learning_rate': 1.0542974530180327e-07, 'num_tokens': 114709.0, 'completions/mean_length': 20.40625, 'completions/min_length': 18.0, 'completions/max_length': 23.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 20.40625, 'completions/min_terminated_length': 18.0, 'completions/max_terminated_length': 23.0, 'rewards/countdown_reward/mean': 0.0, 'rewards/countdown_reward/std': 0.0, 'rewards/format_reward/mean': 0.05937500298023224, 'rewards/format_reward/std': 0.04989909008145332, 'rewards/valid_numbers_reward/mean': 0.10625000298023224, 'rewards/valid_numbers_reward/std': 0.10140147060155869, 'rewards/proximity_reward/mean': 0.02578124962747097, 'rewards/proximity_reward/std': 0.03299827501177788, 'reward': 0.19140625, 'reward_std': 0.1729872077703476, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.022636547684669495, 'sampling/sampling_logp_difference/max': 1.0278531312942505, 'sampling/importance_sampling_ratio/min': 0.17523153126239777, 'sampling/importance_sampling_ratio/mean': 0.909177303314209, 'sampling/importance_sampling_ratio/max': 1.5272537469863892, 'kl': 0.0036370805464684963, 'entropy': 0.07416604459285736, 'clip_ratio/low_mean': 0.0015313936164602637, 'clip_ratio/low_min': 0.0015313936164602637, 'clip_ratio/high_mean': 0.016845328733325005, 'clip_ratio/high_max': 0.016845328733325005, 'clip_ratio/region_mean': 0.018376722931861877, 'step_time': 0.6983875639998587, 'epoch': 1.03125}
54
+ {'loss': 0.1097, 'grad_norm': 5.1096696853637695, 'learning_rate': 8.141676086873573e-08, 'num_tokens': 118175.0, 'completions/mean_length': 20.8125, 'completions/min_length': 18.0, 'completions/max_length': 28.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 20.8125, 'completions/min_terminated_length': 18.0, 'completions/max_terminated_length': 28.0, 'rewards/countdown_reward/mean': 0.21875, 'rewards/countdown_reward/std': 0.420013427734375, 'rewards/format_reward/mean': 0.08750000596046448, 'rewards/format_reward/std': 0.033601075410842896, 'rewards/valid_numbers_reward/mean': 0.16875000298023224, 'rewards/valid_numbers_reward/std': 0.07378040999174118, 'rewards/proximity_reward/mean': 0.07609374821186066, 'rewards/proximity_reward/std': 0.06935650110244751, 'reward': 0.5510936975479126, 'reward_std': 0.5241543054580688, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.01665877178311348, 'sampling/sampling_logp_difference/max': 0.52751624584198, 'sampling/importance_sampling_ratio/min': 0.5610782504081726, 'sampling/importance_sampling_ratio/mean': 0.935814380645752, 'sampling/importance_sampling_ratio/max': 1.783529281616211, 'kl': 0.0037688622251152992, 'entropy': 0.11135981231927872, 'clip_ratio/low_mean': 0.0030030030757188797, 'clip_ratio/low_min': 0.0030030030757188797, 'clip_ratio/high_mean': 0.006006006151437759, 'clip_ratio/high_max': 0.006006006151437759, 'clip_ratio/region_mean': 0.009009009227156639, 'step_time': 0.7239749829998345, 'epoch': 1.0625}
55
+ {'loss': 0.0727, 'grad_norm': 3.130201816558838, 'learning_rate': 6.026312439675551e-08, 'num_tokens': 121518.0, 'completions/mean_length': 16.96875, 'completions/min_length': 12.0, 'completions/max_length': 22.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 16.96875, 'completions/min_terminated_length': 12.0, 'completions/max_terminated_length': 22.0, 'rewards/countdown_reward/mean': 0.34375, 'rewards/countdown_reward/std': 0.4825586974620819, 'rewards/format_reward/mean': 0.09062500298023224, 'rewards/format_reward/std': 0.029614459723234177, 'rewards/valid_numbers_reward/mean': 0.08750000596046448, 'rewards/valid_numbers_reward/std': 0.10080322623252869, 'rewards/proximity_reward/mean': 0.07172618806362152, 'rewards/proximity_reward/std': 0.09507618099451065, 'reward': 0.5936011672019958, 'reward_std': 0.6709741950035095, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.008752727881073952, 'sampling/sampling_logp_difference/max': 0.7403793334960938, 'sampling/importance_sampling_ratio/min': 0.6212528347969055, 'sampling/importance_sampling_ratio/mean': 1.0003535747528076, 'sampling/importance_sampling_ratio/max': 1.7464821338653564, 'kl': 0.0008046461152844131, 'entropy': 0.04473040997982025, 'clip_ratio/low_mean': 0.003683241317048669, 'clip_ratio/low_min': 0.003683241317048669, 'clip_ratio/high_mean': 0.0018416206585243344, 'clip_ratio/high_max': 0.0018416206585243344, 'clip_ratio/region_mean': 0.005524862091988325, 'step_time': 0.6803748840000026, 'epoch': 1.09375}
56
+ {'loss': 0.017, 'grad_norm': 1.5974429845809937, 'learning_rate': 4.2113336672471245e-08, 'num_tokens': 124944.0, 'completions/mean_length': 18.5625, 'completions/min_length': 12.0, 'completions/max_length': 24.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 18.5625, 'completions/min_terminated_length': 12.0, 'completions/max_terminated_length': 24.0, 'rewards/countdown_reward/mean': 0.5, 'rewards/countdown_reward/std': 0.5080004930496216, 'rewards/format_reward/mean': 0.09062500298023224, 'rewards/format_reward/std': 0.029614459723234177, 'rewards/valid_numbers_reward/mean': 0.10625000298023224, 'rewards/valid_numbers_reward/std': 0.10140147060155869, 'rewards/proximity_reward/mean': 0.1026785746216774, 'rewards/proximity_reward/std': 0.09999588131904602, 'reward': 0.7995535731315613, 'reward_std': 0.7141795754432678, 'frac_reward_zero_std': 0.5, 'sampling/sampling_logp_difference/mean': 0.008274354040622711, 'sampling/sampling_logp_difference/max': 0.4873225688934326, 'sampling/importance_sampling_ratio/min': 0.6722076535224915, 'sampling/importance_sampling_ratio/mean': 0.9889246821403503, 'sampling/importance_sampling_ratio/max': 1.6599711179733276, 'kl': 0.00030721016810275614, 'entropy': 0.0578792467713356, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.001683501643128693, 'clip_ratio/high_max': 0.001683501643128693, 'clip_ratio/region_mean': 0.001683501643128693, 'step_time': 0.6986799229998724, 'epoch': 1.125}wandb: updating run metadata
57
+ wandb: uploading output.log; uploading wandb-summary.json; uploading config.yaml
58
+ wandb: uploading wandb-summary.json; uploading config.yaml
59
+ wandb:
60
+ wandb: Run history:
61
+ wandb: profiling/Time taken: GRPOTrainer._calculate_rewards ▃▂▂▂▂▂▂▂▁▂▂▂▂▂▂▂▂▂▂▂▂▁▂▁▂▂▂▂▂▂▂▂▂▂▂▂▁█▂▂
62
+ wandb: profiling/Time taken: GRPOTrainer._get_per_token_logps_and_entropies █▁▃▃▃▂▃▂▃▂▂▃▂▃▂▃▂▂▃▂▃▂▃▂▂▃▃▃▃▂▂▃▃▂▃▃▃▂▂▃
63
+ wandb: profiling/Time taken: GRPOTrainer._prepare_inputs █▁▁▁▁▁▁▁▁▁▁▁▁▅▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
64
+ wandb: profiling/Time taken: GRPOTrainer.compute_loss ▁▁▁▁▁▁▁▁▁▁▁▁▁█▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
65
+ wandb: profiling/Time taken: GRPOTrainer.countdown_reward █▂▂▃▂▂▂▂▂▂▂▂▂▃▂▃▂▂▂▂▂▂▂▂▃▂▂▃▂▂▂▃▂▃▂▂▁▂▂▃
66
+ wandb: profiling/Time taken: GRPOTrainer.format_reward ▆▆▆█▅▆▆▆▃▆▅▆▇▄▅▇▅▆▅▇▆▂▇▃▇▅▄█▆▆▆█▇▇▆▅▁▅▆▆
67
+ wandb: profiling/Time taken: GRPOTrainer.proximity_reward ▁▂▁▁▁▁▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▂▁▁▁▁▁▁▁▁▁▁▁▁█▁▁
68
+ wandb: profiling/Time taken: GRPOTrainer.sync_weights █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
69
+ wandb: profiling/Time taken: GRPOTrainer.vLLM.generate █▁▁▁▁▁▁▁▁▁▁▁▁▄▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
70
+ wandb: profiling/Time taken: GRPOTrainer.valid_numbers_reward ▅▅▄▆▅▅▅▅▃▅▄▄▅▃▄▆▃▅▄▅▅▃▇▂▅▅▄▇▅▇▆▆█▅▄▄▁▄▅▇
71
+ wandb: +37 ...
72
+ wandb:
73
+ wandb: Run summary:
74
+ wandb: profiling/Time taken: GRPOTrainer._calculate_rewards 0.00308
75
+ wandb: profiling/Time taken: GRPOTrainer._get_per_token_logps_and_entropies 0.06604
76
+ wandb: profiling/Time taken: GRPOTrainer._prepare_inputs 0.40072
77
+ wandb: profiling/Time taken: GRPOTrainer.compute_loss 0.09894
78
+ wandb: profiling/Time taken: GRPOTrainer.countdown_reward 0.00069
79
+ wandb: profiling/Time taken: GRPOTrainer.format_reward 0.00056
80
+ wandb: profiling/Time taken: GRPOTrainer.proximity_reward 0.00068
81
+ wandb: profiling/Time taken: GRPOTrainer.sync_weights 0.09454
82
+ wandb: profiling/Time taken: GRPOTrainer.vLLM.generate 0.12266
83
+ wandb: profiling/Time taken: GRPOTrainer.valid_numbers_reward 0.00061
84
+ wandb: +42 ...
85
+ wandb:
86
+ wandb: 🚀 View run smoke_continuous_continuous at: https://wandb.ai/kishanvavdara/llm-zero-lite-smoke/runs/1iawvj3t
87
+ wandb: ⭐️ View project at: https://wandb.ai/kishanvavdara/llm-zero-lite-smoke
88
+ wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)
89
+ wandb: Find logs at: ./wandb/run-20260621_104246-1iawvj3t/logs
90
+
91
+ {'loss': 0.0001, 'grad_norm': 0.052407167851924896, 'learning_rate': 2.7091379149682682e-08, 'num_tokens': 128513.0, 'completions/mean_length': 23.53125, 'completions/min_length': 22.0, 'completions/max_length': 29.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 23.53125, 'completions/min_terminated_length': 22.0, 'completions/max_terminated_length': 29.0, 'rewards/countdown_reward/mean': 0.0, 'rewards/countdown_reward/std': 0.0, 'rewards/format_reward/mean': 0.0, 'rewards/format_reward/std': 0.0, 'rewards/valid_numbers_reward/mean': 0.0, 'rewards/valid_numbers_reward/std': 0.0, 'rewards/proximity_reward/mean': 0.0, 'rewards/proximity_reward/std': 0.0, 'reward': 0.0, 'reward_std': 0.0, 'frac_reward_zero_std': 1.0, 'sampling/sampling_logp_difference/mean': 0.007070348598062992, 'sampling/sampling_logp_difference/max': 1.1611120700836182, 'sampling/importance_sampling_ratio/min': 0.3940061330795288, 'sampling/importance_sampling_ratio/mean': 1.020906686782837, 'sampling/importance_sampling_ratio/max': 1.3242827653884888, 'kl': 0.0013774223625659943, 'entropy': 0.030706292018294334, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 0.7337732739999865, 'epoch': 1.15625}
92
+ {'loss': -0.0881, 'grad_norm': 1.7202657461166382, 'learning_rate': 1.5299867030334813e-08, 'num_tokens': 131803.0, 'completions/mean_length': 15.8125, 'completions/min_length': 12.0, 'completions/max_length': 19.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 15.8125, 'completions/min_terminated_length': 12.0, 'completions/max_terminated_length': 19.0, 'rewards/countdown_reward/mean': 0.0, 'rewards/countdown_reward/std': 0.0, 'rewards/format_reward/mean': 0.10000000149011612, 'rewards/format_reward/std': 0.0, 'rewards/valid_numbers_reward/mean': 0.125, 'rewards/valid_numbers_reward/std': 0.09837387502193451, 'rewards/proximity_reward/mean': 0.040744051337242126, 'rewards/proximity_reward/std': 0.03569040447473526, 'reward': 0.26574403047561646, 'reward_std': 0.13137724995613098, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.00469952542334795, 'sampling/sampling_logp_difference/max': 0.21667885780334473, 'sampling/importance_sampling_ratio/min': 0.6875080466270447, 'sampling/importance_sampling_ratio/mean': 1.0013912916183472, 'sampling/importance_sampling_ratio/max': 1.2280179262161255, 'kl': 0.00043047123472206295, 'entropy': 0.05823312699794769, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.013833992183208466, 'clip_ratio/high_max': 0.013833992183208466, 'clip_ratio/region_mean': 0.013833992183208466, 'step_time': 0.6611508340001819, 'epoch': 1.1875}
93
+ {'loss': -0.0592, 'grad_norm': 1.1424243450164795, 'learning_rate': 6.819348298638839e-09, 'num_tokens': 135334.0, 'completions/mean_length': 20.84375, 'completions/min_length': 18.0, 'completions/max_length': 26.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 20.84375, 'completions/min_terminated_length': 18.0, 'completions/max_terminated_length': 26.0, 'rewards/countdown_reward/mean': 0.96875, 'rewards/countdown_reward/std': 0.1767766922712326, 'rewards/format_reward/mean': 0.10000000149011612, 'rewards/format_reward/std': 0.0, 'rewards/valid_numbers_reward/mean': 0.19374999403953552, 'rewards/valid_numbers_reward/std': 0.0353553406894207, 'rewards/proximity_reward/mean': 0.19374999403953552, 'rewards/proximity_reward/std': 0.0353553406894207, 'reward': 1.4562499523162842, 'reward_std': 0.2474873661994934, 'frac_reward_zero_std': 0.5, 'sampling/sampling_logp_difference/mean': 0.00784258358180523, 'sampling/sampling_logp_difference/max': 0.9508624076843262, 'sampling/importance_sampling_ratio/min': 0.4590398967266083, 'sampling/importance_sampling_ratio/mean': 1.0131592750549316, 'sampling/importance_sampling_ratio/max': 2.6269302368164062, 'kl': 0.0019472006242722273, 'entropy': 0.05225186049938202, 'clip_ratio/low_mean': 0.0014992504147812724, 'clip_ratio/low_min': 0.0014992504147812724, 'clip_ratio/high_mean': 0.00599700165912509, 'clip_ratio/high_max': 0.00599700165912509, 'clip_ratio/region_mean': 0.007496251724660397, 'step_time': 0.716618513999947, 'epoch': 1.21875}
94
+ {'loss': -0.0239, 'grad_norm': 2.2067079544067383, 'learning_rate': 1.7077534966650765e-09, 'num_tokens': 138807.0, 'completions/mean_length': 20.53125, 'completions/min_length': 18.0, 'completions/max_length': 23.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 20.53125, 'completions/min_terminated_length': 18.0, 'completions/max_terminated_length': 23.0, 'rewards/countdown_reward/mean': 0.625, 'rewards/countdown_reward/std': 0.49186936020851135, 'rewards/format_reward/mean': 0.10000000149011612, 'rewards/format_reward/std': 0.0, 'rewards/valid_numbers_reward/mean': 0.17500001192092896, 'rewards/valid_numbers_reward/std': 0.06720215082168579, 'rewards/proximity_reward/mean': 0.1411931812763214, 'rewards/proximity_reward/std': 0.08057914674282074, 'reward': 1.0411932468414307, 'reward_std': 0.6069347858428955, 'frac_reward_zero_std': 0.5, 'sampling/sampling_logp_difference/mean': 0.010136268101632595, 'sampling/sampling_logp_difference/max': 0.7177395820617676, 'sampling/importance_sampling_ratio/min': 0.6189467310905457, 'sampling/importance_sampling_ratio/mean': 0.9915159940719604, 'sampling/importance_sampling_ratio/max': 1.6101794242858887, 'kl': 0.0008815153851173818, 'entropy': 0.06015819311141968, 'clip_ratio/low_mean': 0.006088280119001865, 'clip_ratio/low_min': 0.006088280119001865, 'clip_ratio/high_mean': 0.0015220700297504663, 'clip_ratio/high_max': 0.0015220700297504663, 'clip_ratio/region_mean': 0.007610349915921688, 'step_time': 0.6921010940000087, 'epoch': 1.25}
95
+ {'train_runtime': 37.5295, 'train_samples_per_second': 34.106, 'train_steps_per_second': 1.066, 'train_loss': 0.00969338510142279, 'epoch': 1.25}
96
+
97
+ {
98
+ "method": "continuous_grpo",
99
+ "run_name": "smoke_continuous",
100
+ "stage": 0,
101
+ "global_train_steps": 20,
102
+ "train_reward_mean": 0.5614409944042563,
103
+ "train_reward_std": 0.4356262283399701,
104
+ "kl_mean": 0.0013832212032866664,
105
+ "entropy_mean": 0.0711977096274495,
106
+ "avg_completion_length": 20.796875,
107
+ "completion_length_clip_ratio": 0.0,
108
+ "grad_norm": 3.41826611161232,
109
+ "last_loss": -0.1065,
110
+ "end_learning_rate": 5.82297295140367e-07,
111
+ "eval_accuracy": 0.20833333333333334,
112
+ "eval_greedy_accuracy": 0.20833333333333334,
113
+ "eval_sampled_pass_at_1": 0.25,
114
+ "eval_sampled_pass_at_4": 0.25,
115
+ "eval_avg_completion_length": 19.083333333333332,
116
+ "eval_num_samples": 4,
117
+ "eval_temperature": 1.0,
118
+ "wall_clock_seconds": 37.57142972946167,
119
+ "learning_rate": 1e-06,
120
+ "beta": 0.04,
121
+ "temperature": 1.0,
122
+ "max_completion_length": 512,
123
+ "num_generations": 16
124
+ }
125
+
126
+ {
127
+ "method": "continuous_grpo",
128
+ "run_name": "smoke_continuous",
129
+ "stage": 1,
130
+ "global_train_steps": 40,
131
+ "train_reward_mean": 0.535277726687491,
132
+ "train_reward_std": 0.3741546288132668,
133
+ "kl_mean": 0.001306156341888709,
134
+ "entropy_mean": 0.05558955175802112,
135
+ "avg_completion_length": 20.0140625,
136
+ "completion_length_clip_ratio": 0.0,
137
+ "grad_norm": 2.409226834657602,
138
+ "last_loss": -0.0239,
139
+ "end_learning_rate": 1.7077534966650765e-09,
140
+ "eval_accuracy": 0.20833333333333334,
141
+ "eval_greedy_accuracy": 0.20833333333333334,
142
+ "eval_sampled_pass_at_1": 0.20833333333333334,
143
+ "eval_sampled_pass_at_4": 0.25,
144
+ "eval_avg_completion_length": 19.083333333333332,
145
+ "eval_num_samples": 4,
146
+ "eval_temperature": 1.0,
147
+ "wall_clock_seconds": 37.57142972946167,
148
+ "learning_rate": 1e-06,
149
+ "beta": 0.04,
150
+ "temperature": 1.0,
151
+ "max_completion_length": 512,
152
+ "num_generations": 16
153
+ }
154
+ [rank0]:[W621 10:43:35.981436293 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
runs/smoke_continuous/baseline_metrics.json ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "method": "continuous_grpo",
3
+ "run_name": "smoke_continuous",
4
+ "stage": -1,
5
+ "global_train_steps": 0,
6
+ "eval_accuracy": 0.20833333333333334,
7
+ "eval_greedy_accuracy": 0.20833333333333334,
8
+ "eval_sampled_pass_at_1": 0.16666666666666666,
9
+ "eval_sampled_pass_at_4": 0.25,
10
+ "eval_avg_completion_length": 19.125,
11
+ "eval_num_samples": 4,
12
+ "eval_temperature": 1.0,
13
+ "learning_rate": 1e-06,
14
+ "beta": 0.04,
15
+ "temperature": 1.0,
16
+ "max_completion_length": 512,
17
+ "num_generations": 16
18
+ }
runs/smoke_continuous/experiment_config.json ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_name": "Qwen/Qwen3-1.7B",
3
+ "train_size": 64,
4
+ "eval_size": 24,
5
+ "num_numbers": 3,
6
+ "disable_thinking": true,
7
+ "num_stages": 2,
8
+ "steps_per_stage": 20,
9
+ "seed": 42,
10
+ "learning_rate": 1e-06,
11
+ "lr_scheduler_type": "cosine",
12
+ "warmup_ratio": 0.03,
13
+ "beta": 0.04,
14
+ "temperature": 1.0,
15
+ "max_prompt_length": 256,
16
+ "max_completion_length": 512,
17
+ "num_generations": 16,
18
+ "per_device_train_batch_size": 32,
19
+ "gradient_accumulation_steps": 1,
20
+ "use_vllm": true,
21
+ "vllm_mode": "colocate",
22
+ "vllm_gpu_memory_utilization": 0.55,
23
+ "vllm_enable_sleep_mode": false,
24
+ "vllm_importance_sampling_correction": true,
25
+ "vllm_max_model_length": 768,
26
+ "vllm_tensor_parallel_size": 1,
27
+ "eval_batch_size": 8,
28
+ "eval_num_samples": 4,
29
+ "eval_temperature": 1.0,
30
+ "lora_r": 32,
31
+ "lora_alpha": 64,
32
+ "lora_dropout": 0.05,
33
+ "wandb_project": "llm-zero-lite-smoke",
34
+ "llm_controller_mode": "mock",
35
+ "llm_controller_fail_open": true,
36
+ "llm_controller_max_retries": 3,
37
+ "llm_controller_model": "z-ai/glm-5.2"
38
+ }
runs/smoke_continuous/history.csv ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ method,run_name,stage,global_train_steps,eval_accuracy,eval_greedy_accuracy,eval_sampled_pass_at_1,eval_sampled_pass_at_4,eval_avg_completion_length,eval_num_samples,eval_temperature,learning_rate,beta,temperature,max_completion_length,num_generations,train_reward_mean,train_reward_std,kl_mean,entropy_mean,avg_completion_length,completion_length_clip_ratio,grad_norm,last_loss,end_learning_rate,wall_clock_seconds
2
+ continuous_grpo,smoke_continuous,-1,0,0.20833333333333334,0.20833333333333334,0.16666666666666666,0.25,19.125,4,1.0,1e-06,0.04,1.0,512,16,,,,,,,,,,
3
+ continuous_grpo,smoke_continuous,0,20,0.20833333333333334,0.20833333333333334,0.25,0.25,19.083333333333332,4,1.0,1e-06,0.04,1.0,512,16,0.5614409944042563,0.4356262283399701,0.0013832212032866664,0.0711977096274495,20.796875,0.0,3.41826611161232,-0.1065,5.82297295140367e-07,37.57142972946167
4
+ continuous_grpo,smoke_continuous,1,40,0.20833333333333334,0.20833333333333334,0.20833333333333334,0.25,19.083333333333332,4,1.0,1e-06,0.04,1.0,512,16,0.535277726687491,0.3741546288132668,0.001306156341888709,0.05558955175802112,20.0140625,0.0,2.409226834657602,-0.0239,1.7077534966650765e-09,37.57142972946167
runs/smoke_continuous/train_log.jsonl ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"step": 1, "loss": 0.0109, "grad_norm": 6.041065692901611, "learning_rate": 0.0, "num_tokens": 3425.0, "completions/mean_length": 19.53125, "completions/min_length": 18.0, "completions/max_length": 30.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 19.53125, "completions/min_terminated_length": 18.0, "completions/max_terminated_length": 30.0, "rewards/countdown_reward/mean": 0.59375, "rewards/countdown_reward/std": 0.49899089336395264, "rewards/format_reward/mean": 0.09375, "rewards/format_reward/std": 0.024593466892838478, "rewards/valid_numbers_reward/mean": 0.1875, "rewards/valid_numbers_reward/std": 0.049186933785676956, "rewards/proximity_reward/mean": 0.13208332657814026, "rewards/proximity_reward/std": 0.08473340421915054, "reward": 1.0070834159851074, "reward_std": 0.6107084155082703, "frac_reward_zero_std": 0.0, "sampling/sampling_logp_difference/mean": 0.014617112465202808, "sampling/sampling_logp_difference/max": 0.44372355937957764, "sampling/importance_sampling_ratio/min": 0.4675312638282776, "sampling/importance_sampling_ratio/mean": 1.0141446590423584, "sampling/importance_sampling_ratio/max": 1.5144426822662354, "kl": 0.0, "entropy": 0.10277800261974335, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/high_max": 0.0, "clip_ratio/region_mean": 0.0, "step_time": 3.755320645999973, "epoch": 0.03125}
2
+ {"step": 2, "loss": 0.0458, "grad_norm": 4.3092169761657715, "learning_rate": 5e-07, "num_tokens": 6857.0, "completions/mean_length": 19.25, "completions/min_length": 12.0, "completions/max_length": 24.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 19.25, "completions/min_terminated_length": 12.0, "completions/max_terminated_length": 24.0, "rewards/countdown_reward/mean": 0.03125, "rewards/countdown_reward/std": 0.1767766922712326, "rewards/format_reward/mean": 0.10000000149011612, "rewards/format_reward/std": 0.0, "rewards/valid_numbers_reward/mean": 0.16249999403953552, "rewards/valid_numbers_reward/std": 0.0793115496635437, "rewards/proximity_reward/mean": 0.07604166865348816, "rewards/proximity_reward/std": 0.044184666126966476, "reward": 0.3697916567325592, "reward_std": 0.2362125664949417, "frac_reward_zero_std": 0.0, "sampling/sampling_logp_difference/mean": 0.01406970713287592, "sampling/sampling_logp_difference/max": 0.6686638593673706, "sampling/importance_sampling_ratio/min": 0.4218193590641022, "sampling/importance_sampling_ratio/mean": 1.0178477764129639, "sampling/importance_sampling_ratio/max": 1.8781459331512451, "kl": 0.0, "entropy": 0.06965943425893784, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/high_max": 0.0, "clip_ratio/region_mean": 0.0, "step_time": 0.7183536330001061, "epoch": 0.0625}
3
+ {"step": 3, "loss": -0.0573, "grad_norm": 2.5476601123809814, "learning_rate": 1e-06, "num_tokens": 10398.0, "completions/mean_length": 22.15625, "completions/min_length": 18.0, "completions/max_length": 23.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 22.15625, "completions/min_terminated_length": 18.0, "completions/max_terminated_length": 23.0, "rewards/countdown_reward/mean": 0.21875, "rewards/countdown_reward/std": 0.420013427734375, "rewards/format_reward/mean": 0.05000000074505806, "rewards/format_reward/std": 0.05080005154013634, "rewards/valid_numbers_reward/mean": 0.08750000596046448, "rewards/valid_numbers_reward/std": 0.10080322623252869, "rewards/proximity_reward/mean": 0.04661458730697632, "rewards/proximity_reward/std": 0.08285506069660187, "reward": 0.40286460518836975, "reward_std": 0.6027826070785522, "frac_reward_zero_std": 0.5, "sampling/sampling_logp_difference/mean": 0.014005242846906185, "sampling/sampling_logp_difference/max": 1.7997989654541016, "sampling/importance_sampling_ratio/min": 0.15099729597568512, "sampling/importance_sampling_ratio/mean": 1.0347709655761719, "sampling/importance_sampling_ratio/max": 2.864128351211548, "kl": 0.001527672982774675, "entropy": 0.0723511204123497, "clip_ratio/low_mean": 0.007052185945212841, "clip_ratio/low_min": 0.007052185945212841, "clip_ratio/high_mean": 0.0, "clip_ratio/high_max": 0.0, "clip_ratio/region_mean": 0.007052185945212841, "step_time": 0.7124838239999463, "epoch": 0.09375}
4
+ {"step": 4, "loss": 0.009, "grad_norm": 2.3399267196655273, "learning_rate": 9.982922465033348e-07, "num_tokens": 13925.0, "completions/mean_length": 22.21875, "completions/min_length": 18.0, "completions/max_length": 26.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 22.21875, "completions/min_terminated_length": 18.0, "completions/max_terminated_length": 26.0, "rewards/countdown_reward/mean": 0.71875, "rewards/countdown_reward/std": 0.45680341124534607, "rewards/format_reward/mean": 0.10000000149011612, "rewards/format_reward/std": 0.0, "rewards/valid_numbers_reward/mean": 0.1875, "rewards/valid_numbers_reward/std": 0.049186933785676956, "rewards/proximity_reward/mean": 0.15000000596046448, "rewards/proximity_reward/std": 0.08146130293607712, "reward": 1.15625, "reward_std": 0.5606581568717957, "frac_reward_zero_std": 0.5, "sampling/sampling_logp_difference/mean": 0.0076187350787222385, "sampling/sampling_logp_difference/max": 0.7177844047546387, "sampling/importance_sampling_ratio/min": 0.6298430562019348, "sampling/importance_sampling_ratio/mean": 1.050743579864502, "sampling/importance_sampling_ratio/max": 2.0516271591186523, "kl": 0.001772764720954001, "entropy": 0.05113045871257782, "clip_ratio/low_mean": 0.002812939463183284, "clip_ratio/low_min": 0.002812939463183284, "clip_ratio/high_mean": 0.001406469731591642, "clip_ratio/high_max": 0.001406469731591642, "clip_ratio/region_mean": 0.004219409078359604, "step_time": 0.7690993129999697, "epoch": 0.125}
5
+ {"step": 5, "loss": -0.0315, "grad_norm": 3.5514087677001953, "learning_rate": 9.931806517013612e-07, "num_tokens": 17452.0, "completions/mean_length": 21.21875, "completions/min_length": 18.0, "completions/max_length": 24.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 21.21875, "completions/min_terminated_length": 18.0, "completions/max_terminated_length": 24.0, "rewards/countdown_reward/mean": 0.53125, "rewards/countdown_reward/std": 0.507007360458374, "rewards/format_reward/mean": 0.08437500149011612, "rewards/format_reward/std": 0.03689020499587059, "rewards/valid_numbers_reward/mean": 0.16875000298023224, "rewards/valid_numbers_reward/std": 0.07378040999174118, "rewards/proximity_reward/mean": 0.12217411398887634, "rewards/proximity_reward/std": 0.08884944021701813, "reward": 0.9065490961074829, "reward_std": 0.6525263786315918, "frac_reward_zero_std": 0.0, "sampling/sampling_logp_difference/mean": 0.023038361221551895, "sampling/sampling_logp_difference/max": 0.7486381530761719, "sampling/importance_sampling_ratio/min": 0.0, "sampling/importance_sampling_ratio/mean": 0.9587581157684326, "sampling/importance_sampling_ratio/max": 1.7389065027236938, "kl": 0.0046511460095644, "entropy": 0.06443575024604797, "clip_ratio/low_mean": 0.013254786841571331, "clip_ratio/low_min": 0.013254786841571331, "clip_ratio/high_mean": 0.005891016218811274, "clip_ratio/high_max": 0.005891016218811274, "clip_ratio/region_mean": 0.019145803526043892, "step_time": 0.7193690340000103, "epoch": 0.15625}
6
+ {"step": 6, "loss": 0.0378, "grad_norm": 2.291996955871582, "learning_rate": 9.847001329696652e-07, "num_tokens": 20879.0, "completions/mean_length": 19.59375, "completions/min_length": 18.0, "completions/max_length": 22.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 19.59375, "completions/min_terminated_length": 18.0, "completions/max_terminated_length": 22.0, "rewards/countdown_reward/mean": 0.0, "rewards/countdown_reward/std": 0.0, "rewards/format_reward/mean": 0.10000000149011612, "rewards/format_reward/std": 0.0, "rewards/valid_numbers_reward/mean": 0.20000000298023224, "rewards/valid_numbers_reward/std": 0.0, "rewards/proximity_reward/mean": 0.054739583283662796, "rewards/proximity_reward/std": 0.04368418827652931, "reward": 0.35473960638046265, "reward_std": 0.043684203177690506, "frac_reward_zero_std": 0.0, "sampling/sampling_logp_difference/mean": 0.01503602135926485, "sampling/sampling_logp_difference/max": 0.6983115673065186, "sampling/importance_sampling_ratio/min": 0.49442461133003235, "sampling/importance_sampling_ratio/mean": 1.021022081375122, "sampling/importance_sampling_ratio/max": 1.8775709867477417, "kl": 0.0014773447765037417, "entropy": 0.07872910797595978, "clip_ratio/low_mean": 0.003189792623743415, "clip_ratio/low_min": 0.003189792623743415, "clip_ratio/high_mean": 0.0047846888191998005, "clip_ratio/high_max": 0.0047846888191998005, "clip_ratio/region_mean": 0.007974481210112572, "step_time": 0.6970211140001084, "epoch": 0.1875}
7
+ {"step": 7, "loss": 0.0806, "grad_norm": 3.5400500297546387, "learning_rate": 9.729086208503173e-07, "num_tokens": 24397.0, "completions/mean_length": 21.4375, "completions/min_length": 18.0, "completions/max_length": 25.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 21.4375, "completions/min_terminated_length": 18.0, "completions/max_terminated_length": 25.0, "rewards/countdown_reward/mean": 0.03125, "rewards/countdown_reward/std": 0.1767766922712326, "rewards/format_reward/mean": 0.09375, "rewards/format_reward/std": 0.024593466892838478, "rewards/valid_numbers_reward/mean": 0.1875, "rewards/valid_numbers_reward/std": 0.049186933785676956, "rewards/proximity_reward/mean": 0.026139114052057266, "rewards/proximity_reward/std": 0.03372207283973694, "reward": 0.33863911032676697, "reward_std": 0.22642673552036285, "frac_reward_zero_std": 0.0, "sampling/sampling_logp_difference/mean": 0.00891045667231083, "sampling/sampling_logp_difference/max": 0.3650592565536499, "sampling/importance_sampling_ratio/min": 0.548480212688446, "sampling/importance_sampling_ratio/mean": 1.071790099143982, "sampling/importance_sampling_ratio/max": 2.0098817348480225, "kl": 0.0005612155073322356, "entropy": 0.05858924612402916, "clip_ratio/low_mean": 0.0029154520016163588, "clip_ratio/low_min": 0.0029154520016163588, "clip_ratio/high_mean": 0.004373177886009216, "clip_ratio/high_max": 0.004373177886009216, "clip_ratio/region_mean": 0.007288629654794931, "step_time": 0.7219514629998685, "epoch": 0.21875}
8
+ {"step": 8, "loss": 0.1165, "grad_norm": 3.065430164337158, "learning_rate": 9.578866633275286e-07, "num_tokens": 27842.0, "completions/mean_length": 19.65625, "completions/min_length": 16.0, "completions/max_length": 23.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 19.65625, "completions/min_terminated_length": 16.0, "completions/max_terminated_length": 23.0, "rewards/countdown_reward/mean": 0.34375, "rewards/countdown_reward/std": 0.4825586974620819, "rewards/format_reward/mean": 0.09375, "rewards/format_reward/std": 0.024593466892838478, "rewards/valid_numbers_reward/mean": 0.17500001192092896, "rewards/valid_numbers_reward/std": 0.06720215082168579, "rewards/proximity_reward/mean": 0.10104167461395264, "rewards/proximity_reward/std": 0.07623975723981857, "reward": 0.7135416269302368, "reward_std": 0.5873492956161499, "frac_reward_zero_std": 0.0, "sampling/sampling_logp_difference/mean": 0.01804327219724655, "sampling/sampling_logp_difference/max": 0.6249973773956299, "sampling/importance_sampling_ratio/min": 0.6445150971412659, "sampling/importance_sampling_ratio/mean": 1.0156548023223877, "sampling/importance_sampling_ratio/max": 2.68859601020813, "kl": 0.0026622372679412365, "entropy": 0.08564577996730804, "clip_ratio/low_mean": 0.007949125953018665, "clip_ratio/low_min": 0.007949125953018665, "clip_ratio/high_mean": 0.007949125953018665, "clip_ratio/high_max": 0.007949125953018665, "clip_ratio/region_mean": 0.01589825190603733, "step_time": 0.7035421840000708, "epoch": 0.25}
9
+ {"step": 9, "loss": -0.0592, "grad_norm": 3.6667890548706055, "learning_rate": 9.397368756032444e-07, "num_tokens": 31347.0, "completions/mean_length": 20.53125, "completions/min_length": 18.0, "completions/max_length": 23.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 20.53125, "completions/min_terminated_length": 18.0, "completions/max_terminated_length": 23.0, "rewards/countdown_reward/mean": 0.5, "rewards/countdown_reward/std": 0.5080004930496216, "rewards/format_reward/mean": 0.05312500149011612, "rewards/format_reward/std": 0.05070073530077934, "rewards/valid_numbers_reward/mean": 0.10625000298023224, "rewards/valid_numbers_reward/std": 0.10140147060155869, "rewards/proximity_reward/mean": 0.10078125447034836, "rewards/proximity_reward/std": 0.10090005397796631, "reward": 0.7601562738418579, "reward_std": 0.7538037896156311, "frac_reward_zero_std": 0.5, "sampling/sampling_logp_difference/mean": 0.0033016959205269814, "sampling/sampling_logp_difference/max": 0.49289751052856445, "sampling/importance_sampling_ratio/min": 0.8354066014289856, "sampling/importance_sampling_ratio/mean": 1.010882019996643, "sampling/importance_sampling_ratio/max": 1.7604669332504272, "kl": 0.0012082780012860894, "entropy": 0.026930633932352066, "clip_ratio/low_mean": 0.0015220700297504663, "clip_ratio/low_min": 0.0015220700297504663, "clip_ratio/high_mean": 0.0, "clip_ratio/high_max": 0.0, "clip_ratio/region_mean": 0.0015220700297504663, "step_time": 0.7023271729999578, "epoch": 0.28125}
10
+ {"step": 10, "loss": 0.0201, "grad_norm": 4.752869606018066, "learning_rate": 9.185832391312642e-07, "num_tokens": 34845.0, "completions/mean_length": 20.8125, "completions/min_length": 18.0, "completions/max_length": 23.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 20.8125, "completions/min_terminated_length": 18.0, "completions/max_terminated_length": 23.0, "rewards/countdown_reward/mean": 0.625, "rewards/countdown_reward/std": 0.49186936020851135, "rewards/format_reward/mean": 0.09375, "rewards/format_reward/std": 0.024593466892838478, "rewards/valid_numbers_reward/mean": 0.1875, "rewards/valid_numbers_reward/std": 0.049186933785676956, "rewards/proximity_reward/mean": 0.13895833492279053, "rewards/proximity_reward/std": 0.08098160475492477, "reward": 1.0452083349227905, "reward_std": 0.6018912196159363, "frac_reward_zero_std": 0.5, "sampling/sampling_logp_difference/mean": 0.014039169996976852, "sampling/sampling_logp_difference/max": 0.6249973773956299, "sampling/importance_sampling_ratio/min": 0.4273960292339325, "sampling/importance_sampling_ratio/mean": 1.032275915145874, "sampling/importance_sampling_ratio/max": 2.3014140129089355, "kl": 0.0016276317182928324, "entropy": 0.07305138558149338, "clip_ratio/low_mean": 0.006006006151437759, "clip_ratio/low_min": 0.006006006151437759, "clip_ratio/high_mean": 0.0015015015378594398, "clip_ratio/high_max": 0.0015015015378594398, "clip_ratio/region_mean": 0.007507507689297199, "step_time": 0.7014926839999589, "epoch": 0.3125}
11
+ {"step": 11, "loss": -0.1784, "grad_norm": 6.029069423675537, "learning_rate": 8.945702546981968e-07, "num_tokens": 38261.0, "completions/mean_length": 18.25, "completions/min_length": 12.0, "completions/max_length": 24.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 18.25, "completions/min_terminated_length": 12.0, "completions/max_terminated_length": 24.0, "rewards/countdown_reward/mean": 0.0, "rewards/countdown_reward/std": 0.0, "rewards/format_reward/mean": 0.10000000149011612, "rewards/format_reward/std": 0.0, "rewards/valid_numbers_reward/mean": 0.15625, "rewards/valid_numbers_reward/std": 0.08400268852710724, "rewards/proximity_reward/mean": 0.04296875, "rewards/proximity_reward/std": 0.040775928646326065, "reward": 0.2992187738418579, "reward_std": 0.11225052177906036, "frac_reward_zero_std": 0.0, "sampling/sampling_logp_difference/mean": 0.014431153424084187, "sampling/sampling_logp_difference/max": 0.5645503997802734, "sampling/importance_sampling_ratio/min": 0.5598352551460266, "sampling/importance_sampling_ratio/mean": 1.0049711465835571, "sampling/importance_sampling_ratio/max": 2.108710527420044, "kl": 0.0014566656900569797, "entropy": 0.08794382214546204, "clip_ratio/low_mean": 0.0017123287543654442, "clip_ratio/low_min": 0.0017123287543654442, "clip_ratio/high_mean": 0.006849315017461777, "clip_ratio/high_max": 0.006849315017461777, "clip_ratio/region_mean": 0.008561643771827221, "step_time": 0.7050704440000573, "epoch": 0.34375}
12
+ {"step": 12, "loss": 0.1242, "grad_norm": 3.043853998184204, "learning_rate": 8.678619553365658e-07, "num_tokens": 41675.0, "completions/mean_length": 18.1875, "completions/min_length": 12.0, "completions/max_length": 25.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 18.1875, "completions/min_terminated_length": 12.0, "completions/max_terminated_length": 25.0, "rewards/countdown_reward/mean": 0.5, "rewards/countdown_reward/std": 0.5080004930496216, "rewards/format_reward/mean": 0.09687499701976776, "rewards/format_reward/std": 0.01767767034471035, "rewards/valid_numbers_reward/mean": 0.10000000149011612, "rewards/valid_numbers_reward/std": 0.10160010308027267, "rewards/proximity_reward/mean": 0.10000000149011612, "rewards/proximity_reward/std": 0.10160010308027267, "reward": 0.796875, "reward_std": 0.7145873308181763, "frac_reward_zero_std": 0.5, "sampling/sampling_logp_difference/mean": 0.007687629200518131, "sampling/sampling_logp_difference/max": 0.5967673063278198, "sampling/importance_sampling_ratio/min": 0.6460556387901306, "sampling/importance_sampling_ratio/mean": 0.9738063812255859, "sampling/importance_sampling_ratio/max": 1.5643600225448608, "kl": 0.0004984450642950833, "entropy": 0.05066652595996857, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/high_mean": 0.0017182130832225084, "clip_ratio/high_max": 0.0017182130832225084, "clip_ratio/region_mean": 0.0017182130832225084, "step_time": 0.7119393329999184, "epoch": 0.375}
13
+ {"step": 13, "loss": -0.0156, "grad_norm": 2.8895351886749268, "learning_rate": 8.386407858128706e-07, "num_tokens": 45144.0, "completions/mean_length": 20.40625, "completions/min_length": 18.0, "completions/max_length": 23.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 20.40625, "completions/min_terminated_length": 18.0, "completions/max_terminated_length": 23.0, "rewards/countdown_reward/mean": 0.5, "rewards/countdown_reward/std": 0.5080004930496216, "rewards/format_reward/mean": 0.10000000149011612, "rewards/format_reward/std": 0.0, "rewards/valid_numbers_reward/mean": 0.20000000298023224, "rewards/valid_numbers_reward/std": 0.0, "rewards/proximity_reward/mean": 0.10451087355613708, "rewards/proximity_reward/std": 0.09701789915561676, "reward": 0.9045108556747437, "reward_std": 0.6050176620483398, "frac_reward_zero_std": 0.5, "sampling/sampling_logp_difference/mean": 0.0066388980485498905, "sampling/sampling_logp_difference/max": 0.3109920620918274, "sampling/importance_sampling_ratio/min": 0.7341673970222473, "sampling/importance_sampling_ratio/mean": 0.9615199565887451, "sampling/importance_sampling_ratio/max": 1.2092349529266357, "kl": 0.000671109592076391, "entropy": 0.050037682056427, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/high_mean": 0.0015313936164602637, "clip_ratio/high_max": 0.0015313936164602637, "clip_ratio/region_mean": 0.0015313936164602637, "step_time": 0.6962026339999738, "epoch": 0.40625}
14
+ {"step": 14, "loss": 0.0331, "grad_norm": 1.8920438289642334, "learning_rate": 8.071063563448339e-07, "num_tokens": 49016.0, "completions/mean_length": 33.0, "completions/min_length": 18.0, "completions/max_length": 307.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 33.0, "completions/min_terminated_length": 18.0, "completions/max_terminated_length": 307.0, "rewards/countdown_reward/mean": 0.0, "rewards/countdown_reward/std": 0.0, "rewards/format_reward/mean": 0.012500000186264515, "rewards/format_reward/std": 0.033601075410842896, "rewards/valid_numbers_reward/mean": 0.02500000037252903, "rewards/valid_numbers_reward/std": 0.06720215082168579, "rewards/proximity_reward/mean": 0.004999999888241291, "rewards/proximity_reward/std": 0.013440429233014584, "reward": 0.042500000447034836, "reward_std": 0.11424365639686584, "frac_reward_zero_std": 0.5, "sampling/sampling_logp_difference/mean": 0.007544555701315403, "sampling/sampling_logp_difference/max": 0.3881516456604004, "sampling/importance_sampling_ratio/min": 0.7685970067977905, "sampling/importance_sampling_ratio/mean": 1.0223689079284668, "sampling/importance_sampling_ratio/max": 1.4428951740264893, "kl": 0.0007539401995018125, "entropy": 0.10343185812234879, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/high_max": 0.0, "clip_ratio/region_mean": 0.0, "step_time": 3.272125610999865, "epoch": 0.4375}
15
+ {"step": 15, "loss": 0.1982, "grad_norm": 4.963200092315674, "learning_rate": 7.734740790612136e-07, "num_tokens": 52421.0, "completions/mean_length": 18.90625, "completions/min_length": 18.0, "completions/max_length": 22.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 18.90625, "completions/min_terminated_length": 18.0, "completions/max_terminated_length": 22.0, "rewards/countdown_reward/mean": 0.0, "rewards/countdown_reward/std": 0.0, "rewards/format_reward/mean": 0.08124999701976776, "rewards/format_reward/std": 0.03965577483177185, "rewards/valid_numbers_reward/mean": 0.16249999403953552, "rewards/valid_numbers_reward/std": 0.0793115496635437, "rewards/proximity_reward/mean": 0.03890354186296463, "rewards/proximity_reward/std": 0.03131883218884468, "reward": 0.2826535403728485, "reward_std": 0.14018532633781433, "frac_reward_zero_std": 0.0, "sampling/sampling_logp_difference/mean": 0.01146075502038002, "sampling/sampling_logp_difference/max": 0.6627943515777588, "sampling/importance_sampling_ratio/min": 0.49324995279312134, "sampling/importance_sampling_ratio/mean": 0.9984955787658691, "sampling/importance_sampling_ratio/max": 1.7686353921890259, "kl": 0.00044077940401621163, "entropy": 0.05800556764006615, "clip_ratio/low_mean": 0.0016528925625607371, "clip_ratio/low_min": 0.0016528925625607371, "clip_ratio/high_mean": 0.00826446246355772, "clip_ratio/high_max": 0.00826446246355772, "clip_ratio/region_mean": 0.00991735514253378, "step_time": 0.6855096929998581, "epoch": 0.46875}
16
+ {"step": 16, "loss": -0.0484, "grad_norm": 2.715698480606079, "learning_rate": 7.379736965185368e-07, "num_tokens": 55881.0, "completions/mean_length": 21.125, "completions/min_length": 18.0, "completions/max_length": 24.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 21.125, "completions/min_terminated_length": 18.0, "completions/max_terminated_length": 24.0, "rewards/countdown_reward/mean": 0.25, "rewards/countdown_reward/std": 0.4399413466453552, "rewards/format_reward/mean": 0.09687499701976776, "rewards/format_reward/std": 0.01767767034471035, "rewards/valid_numbers_reward/mean": 0.10000000149011612, "rewards/valid_numbers_reward/std": 0.10160010308027267, "rewards/proximity_reward/mean": 0.0627967119216919, "rewards/proximity_reward/std": 0.0837658941745758, "reward": 0.509671688079834, "reward_std": 0.5909783244132996, "frac_reward_zero_std": 0.0, "sampling/sampling_logp_difference/mean": 0.015001348219811916, "sampling/sampling_logp_difference/max": 0.7091498374938965, "sampling/importance_sampling_ratio/min": 0.36538299918174744, "sampling/importance_sampling_ratio/mean": 0.9959367513656616, "sampling/importance_sampling_ratio/max": 1.694196105003357, "kl": 0.0013793795369565487, "entropy": 0.10765296965837479, "clip_ratio/low_mean": 0.002958579920232296, "clip_ratio/low_min": 0.002958579920232296, "clip_ratio/high_mean": 0.002958579920232296, "clip_ratio/high_max": 0.002958579920232296, "clip_ratio/region_mean": 0.005917159840464592, "step_time": 0.6983848740001122, "epoch": 0.5}
17
+ {"step": 17, "loss": 0.0272, "grad_norm": 1.209149718284607, "learning_rate": 7.008477123264847e-07, "num_tokens": 59364.0, "completions/mean_length": 21.34375, "completions/min_length": 18.0, "completions/max_length": 29.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 21.34375, "completions/min_terminated_length": 18.0, "completions/max_terminated_length": 29.0, "rewards/countdown_reward/mean": 0.0, "rewards/countdown_reward/std": 0.0, "rewards/format_reward/mean": 0.05000000074505806, "rewards/format_reward/std": 0.05080005154013634, "rewards/valid_numbers_reward/mean": 0.10000000149011612, "rewards/valid_numbers_reward/std": 0.10160010308027267, "rewards/proximity_reward/mean": 0.02812499925494194, "rewards/proximity_reward/std": 0.03094610385596752, "reward": 0.17812499403953552, "reward_std": 0.18136467039585114, "frac_reward_zero_std": 0.5, "sampling/sampling_logp_difference/mean": 0.004375044722110033, "sampling/sampling_logp_difference/max": 0.4909186363220215, "sampling/importance_sampling_ratio/min": 0.5967592597007751, "sampling/importance_sampling_ratio/mean": 0.9605278968811035, "sampling/importance_sampling_ratio/max": 1.3050473928451538, "kl": 0.00023233593674376607, "entropy": 0.03696838766336441, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/high_mean": 0.0014641288435086608, "clip_ratio/high_max": 0.0014641288435086608, "clip_ratio/region_mean": 0.0014641288435086608, "step_time": 0.7314563830000225, "epoch": 0.53125}
18
+ {"step": 18, "loss": 0.0942, "grad_norm": 2.5989675521850586, "learning_rate": 6.623497346023417e-07, "num_tokens": 62793.0, "completions/mean_length": 20.15625, "completions/min_length": 16.0, "completions/max_length": 26.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 20.15625, "completions/min_terminated_length": 16.0, "completions/max_terminated_length": 26.0, "rewards/countdown_reward/mean": 0.0, "rewards/countdown_reward/std": 0.0, "rewards/format_reward/mean": 0.07187500596046448, "rewards/format_reward/std": 0.04568034037947655, "rewards/valid_numbers_reward/mean": 0.11249999701976776, "rewards/valid_numbers_reward/std": 0.10080322623252869, "rewards/proximity_reward/mean": 0.03396707400679588, "rewards/proximity_reward/std": 0.050391241908073425, "reward": 0.21834206581115723, "reward_std": 0.17154528200626373, "frac_reward_zero_std": 0.0, "sampling/sampling_logp_difference/mean": 0.014694616198539734, "sampling/sampling_logp_difference/max": 0.9361472725868225, "sampling/importance_sampling_ratio/min": 0.5324814915657043, "sampling/importance_sampling_ratio/mean": 1.020961046218872, "sampling/importance_sampling_ratio/max": 1.700570821762085, "kl": 0.002194973174482584, "entropy": 0.09666672348976135, "clip_ratio/low_mean": 0.010852713137865067, "clip_ratio/low_min": 0.010852713137865067, "clip_ratio/high_mean": 0.006201550364494324, "clip_ratio/high_max": 0.006201550364494324, "clip_ratio/region_mean": 0.01705426350235939, "step_time": 0.7106611730000623, "epoch": 0.5625}
19
+ {"step": 19, "loss": 0.1011, "grad_norm": 3.349228858947754, "learning_rate": 6.227427435703995e-07, "num_tokens": 66155.0, "completions/mean_length": 17.0625, "completions/min_length": 12.0, "completions/max_length": 28.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 17.0625, "completions/min_terminated_length": 12.0, "completions/max_terminated_length": 28.0, "rewards/countdown_reward/mean": 0.3125, "rewards/countdown_reward/std": 0.4709290862083435, "rewards/format_reward/mean": 0.09062500298023224, "rewards/format_reward/std": 0.029614459723234177, "rewards/valid_numbers_reward/mean": 0.06875000149011612, "rewards/valid_numbers_reward/std": 0.09651173651218414, "rewards/proximity_reward/mean": 0.06302083283662796, "rewards/proximity_reward/std": 0.09387461096048355, "reward": 0.5348958373069763, "reward_std": 0.6629514694213867, "frac_reward_zero_std": 0.5, "sampling/sampling_logp_difference/mean": 0.0075608654879033566, "sampling/sampling_logp_difference/max": 0.730109691619873, "sampling/importance_sampling_ratio/min": 0.7518770694732666, "sampling/importance_sampling_ratio/mean": 1.0364713668823242, "sampling/importance_sampling_ratio/max": 2.0383474826812744, "kl": 0.002250331686809659, "entropy": 0.06416559964418411, "clip_ratio/low_mean": 0.005494505632668734, "clip_ratio/low_min": 0.005494505632668734, "clip_ratio/high_mean": 0.005494505632668734, "clip_ratio/high_max": 0.005494505632668734, "clip_ratio/region_mean": 0.010989011265337467, "step_time": 0.7281739340000968, "epoch": 0.59375}
20
+ {"step": 20, "loss": -0.1065, "grad_norm": 3.5681610107421875, "learning_rate": 5.82297295140367e-07, "num_tokens": 69662.0, "completions/mean_length": 21.09375, "completions/min_length": 19.0, "completions/max_length": 27.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 21.09375, "completions/min_terminated_length": 19.0, "completions/max_terminated_length": 27.0, "rewards/countdown_reward/mean": 0.1875, "rewards/countdown_reward/std": 0.3965577781200409, "rewards/format_reward/mean": 0.08750000596046448, "rewards/format_reward/std": 0.033601075410842896, "rewards/valid_numbers_reward/mean": 0.09375, "rewards/valid_numbers_reward/std": 0.10140147060155869, "rewards/proximity_reward/mean": 0.038453392684459686, "rewards/proximity_reward/std": 0.07886006683111191, "reward": 0.40720340609550476, "reward_std": 0.5433569550514221, "frac_reward_zero_std": 0.0, "sampling/sampling_logp_difference/mean": 0.011155059561133385, "sampling/sampling_logp_difference/max": 0.7159504890441895, "sampling/importance_sampling_ratio/min": 0.5663295388221741, "sampling/importance_sampling_ratio/mean": 1.0255656242370605, "sampling/importance_sampling_ratio/max": 1.3683027029037476, "kl": 0.0022981727961450815, "entropy": 0.08511413633823395, "clip_ratio/low_mean": 0.00296296295709908, "clip_ratio/low_min": 0.00296296295709908, "clip_ratio/high_mean": 0.0, "clip_ratio/high_max": 0.0, "clip_ratio/region_mean": 0.00296296295709908, "step_time": 0.7250004739998985, "epoch": 0.625}
21
+ {"step": 21, "loss": 0.0344, "grad_norm": 1.683887243270874, "learning_rate": 5.412896727361662e-07, "num_tokens": 73220.0, "completions/mean_length": 22.1875, "completions/min_length": 19.0, "completions/max_length": 23.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 22.1875, "completions/min_terminated_length": 19.0, "completions/max_terminated_length": 23.0, "rewards/countdown_reward/mean": 0.5, "rewards/countdown_reward/std": 0.5080004930496216, "rewards/format_reward/mean": 0.10000000149011612, "rewards/format_reward/std": 0.0, "rewards/valid_numbers_reward/mean": 0.20000000298023224, "rewards/valid_numbers_reward/std": 0.0, "rewards/proximity_reward/mean": 0.12166666984558105, "rewards/proximity_reward/std": 0.07983854413032532, "reward": 0.92166668176651, "reward_std": 0.5876213908195496, "frac_reward_zero_std": 0.5, "sampling/sampling_logp_difference/mean": 0.005603067576885223, "sampling/sampling_logp_difference/max": 0.48111605644226074, "sampling/importance_sampling_ratio/min": 0.7776699066162109, "sampling/importance_sampling_ratio/mean": 1.0168145895004272, "sampling/importance_sampling_ratio/max": 1.4217238426208496, "kl": 0.0009874801617115736, "entropy": 0.028706880286335945, "clip_ratio/low_mean": 0.002816901309415698, "clip_ratio/low_min": 0.002816901309415698, "clip_ratio/high_mean": 0.002816901309415698, "clip_ratio/high_max": 0.002816901309415698, "clip_ratio/region_mean": 0.005633802618831396, "step_time": 0.6987281040001108, "epoch": 0.65625}
22
+ {"step": 22, "loss": -0.0805, "grad_norm": 2.383918285369873, "learning_rate": 5e-07, "num_tokens": 76583.0, "completions/mean_length": 17.59375, "completions/min_length": 12.0, "completions/max_length": 23.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 17.59375, "completions/min_terminated_length": 12.0, "completions/max_terminated_length": 23.0, "rewards/countdown_reward/mean": 0.0, "rewards/countdown_reward/std": 0.0, "rewards/format_reward/mean": 0.05000000074505806, "rewards/format_reward/std": 0.05080005154013634, "rewards/valid_numbers_reward/mean": 0.01875000074505806, "rewards/valid_numbers_reward/std": 0.05922891944646835, "rewards/proximity_reward/mean": 0.0026785715017467737, "rewards/proximity_reward/std": 0.008461273275315762, "reward": 0.0714285746216774, "reward_std": 0.09682246297597885, "frac_reward_zero_std": 0.5, "sampling/sampling_logp_difference/mean": 0.0034464546479284763, "sampling/sampling_logp_difference/max": 0.3867391347885132, "sampling/importance_sampling_ratio/min": 0.7479129433631897, "sampling/importance_sampling_ratio/mean": 0.9816645383834839, "sampling/importance_sampling_ratio/max": 1.3836370706558228, "kl": 0.0002953831572085619, "entropy": 0.03434375301003456, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/high_mean": 0.0017761989729478955, "clip_ratio/high_max": 0.0017761989729478955, "clip_ratio/region_mean": 0.0017761989729478955, "step_time": 0.6868111840001347, "epoch": 0.6875}
23
+ {"step": 23, "loss": 0.0335, "grad_norm": 1.8223655223846436, "learning_rate": 4.5871032726383385e-07, "num_tokens": 80047.0, "completions/mean_length": 20.25, "completions/min_length": 18.0, "completions/max_length": 23.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 20.25, "completions/min_terminated_length": 18.0, "completions/max_terminated_length": 23.0, "rewards/countdown_reward/mean": 0.03125, "rewards/countdown_reward/std": 0.1767766922712326, "rewards/format_reward/mean": 0.10000000149011612, "rewards/format_reward/std": 0.0, "rewards/valid_numbers_reward/mean": 0.1875, "rewards/valid_numbers_reward/std": 0.049186933785676956, "rewards/proximity_reward/mean": 0.0259108766913414, "rewards/proximity_reward/std": 0.04473352059721947, "reward": 0.3446608781814575, "reward_std": 0.21994799375534058, "frac_reward_zero_std": 0.0, "sampling/sampling_logp_difference/mean": 0.010006303898990154, "sampling/sampling_logp_difference/max": 0.6734256744384766, "sampling/importance_sampling_ratio/min": 0.568607747554779, "sampling/importance_sampling_ratio/mean": 1.0024545192718506, "sampling/importance_sampling_ratio/max": 2.2461788654327393, "kl": 0.0014518073294311762, "entropy": 0.07266536355018616, "clip_ratio/low_mean": 0.009259259328246117, "clip_ratio/low_min": 0.009259259328246117, "clip_ratio/high_mean": 0.004629629664123058, "clip_ratio/high_max": 0.004629629664123058, "clip_ratio/region_mean": 0.013888888992369175, "step_time": 0.6992476539999188, "epoch": 0.71875}
24
+ {"step": 24, "loss": -0.114, "grad_norm": 5.045168399810791, "learning_rate": 4.1770270485963294e-07, "num_tokens": 83353.0, "completions/mean_length": 16.3125, "completions/min_length": 12.0, "completions/max_length": 21.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 16.3125, "completions/min_terminated_length": 12.0, "completions/max_terminated_length": 21.0, "rewards/countdown_reward/mean": 0.0, "rewards/countdown_reward/std": 0.0, "rewards/format_reward/mean": 0.05312500149011612, "rewards/format_reward/std": 0.05070073530077934, "rewards/valid_numbers_reward/mean": 0.0062500000931322575, "rewards/valid_numbers_reward/std": 0.0353553406894207, "rewards/proximity_reward/mean": 0.000480769231216982, "rewards/proximity_reward/std": 0.0027196414303034544, "reward": 0.05985577031970024, "reward_std": 0.06834935396909714, "frac_reward_zero_std": 0.5, "sampling/sampling_logp_difference/mean": 0.014905628748238087, "sampling/sampling_logp_difference/max": 1.1528494358062744, "sampling/importance_sampling_ratio/min": 0.3555085062980652, "sampling/importance_sampling_ratio/mean": 0.9059621691703796, "sampling/importance_sampling_ratio/max": 1.7516393661499023, "kl": 0.0012164042564108968, "entropy": 0.042202427983284, "clip_ratio/low_mean": 0.005747126415371895, "clip_ratio/low_min": 0.005747126415371895, "clip_ratio/high_mean": 0.0, "clip_ratio/high_max": 0.0, "clip_ratio/region_mean": 0.005747126415371895, "step_time": 0.6691923840000982, "epoch": 0.75}
25
+ {"step": 25, "loss": -0.046, "grad_norm": 3.4389476776123047, "learning_rate": 3.772572564296004e-07, "num_tokens": 86848.0, "completions/mean_length": 21.21875, "completions/min_length": 13.0, "completions/max_length": 23.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 21.21875, "completions/min_terminated_length": 13.0, "completions/max_terminated_length": 23.0, "rewards/countdown_reward/mean": 0.0625, "rewards/countdown_reward/std": 0.24593468010425568, "rewards/format_reward/mean": 0.09062500298023224, "rewards/format_reward/std": 0.029614457860589027, "rewards/valid_numbers_reward/mean": 0.17500001192092896, "rewards/valid_numbers_reward/std": 0.06720215082168579, "rewards/proximity_reward/mean": 0.057149626314640045, "rewards/proximity_reward/std": 0.044902537018060684, "reward": 0.38527464866638184, "reward_std": 0.3133750557899475, "frac_reward_zero_std": 0.0, "sampling/sampling_logp_difference/mean": 0.01222001202404499, "sampling/sampling_logp_difference/max": 0.8772695064544678, "sampling/importance_sampling_ratio/min": 0.5971362590789795, "sampling/importance_sampling_ratio/mean": 1.023521900177002, "sampling/importance_sampling_ratio/max": 2.4219090938568115, "kl": 0.0022281454876065254, "entropy": 0.07401835173368454, "clip_ratio/low_mean": 0.0014727540547028184, "clip_ratio/low_min": 0.0014727540547028184, "clip_ratio/high_mean": 0.011782032437622547, "clip_ratio/high_max": 0.011782032437622547, "clip_ratio/region_mean": 0.013254786841571331, "step_time": 0.6923223940000298, "epoch": 0.78125}
26
+ {"step": 26, "loss": -0.0309, "grad_norm": 1.6091957092285156, "learning_rate": 3.3765026539765827e-07, "num_tokens": 90326.0, "completions/mean_length": 19.6875, "completions/min_length": 18.0, "completions/max_length": 25.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 19.6875, "completions/min_terminated_length": 18.0, "completions/max_terminated_length": 25.0, "rewards/countdown_reward/mean": 0.59375, "rewards/countdown_reward/std": 0.49899089336395264, "rewards/format_reward/mean": 0.10000000149011612, "rewards/format_reward/std": 0.0, "rewards/valid_numbers_reward/mean": 0.20000000298023224, "rewards/valid_numbers_reward/std": 0.0, "rewards/proximity_reward/mean": 0.1383928656578064, "rewards/proximity_reward/std": 0.07889021933078766, "reward": 1.0321428775787354, "reward_std": 0.5750947594642639, "frac_reward_zero_std": 0.5, "sampling/sampling_logp_difference/mean": 0.007631146814674139, "sampling/sampling_logp_difference/max": 0.48116254806518555, "sampling/importance_sampling_ratio/min": 0.6717771887779236, "sampling/importance_sampling_ratio/mean": 1.010103702545166, "sampling/importance_sampling_ratio/max": 1.5968704223632812, "kl": 0.0010196284856647253, "entropy": 0.05027592182159424, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/high_mean": 0.0031746032182127237, "clip_ratio/high_max": 0.0031746032182127237, "clip_ratio/region_mean": 0.0031746032182127237, "step_time": 0.7108742530001564, "epoch": 0.8125}
27
+ {"step": 27, "loss": 0.014, "grad_norm": 1.6168516874313354, "learning_rate": 2.9915228767351535e-07, "num_tokens": 93786.0, "completions/mean_length": 20.125, "completions/min_length": 18.0, "completions/max_length": 22.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 20.125, "completions/min_terminated_length": 18.0, "completions/max_terminated_length": 22.0, "rewards/countdown_reward/mean": 0.15625, "rewards/countdown_reward/std": 0.3689020276069641, "rewards/format_reward/mean": 0.06562499701976776, "rewards/format_reward/std": 0.04825586825609207, "rewards/valid_numbers_reward/mean": 0.13124999403953552, "rewards/valid_numbers_reward/std": 0.09651173651218414, "rewards/proximity_reward/mean": 0.06458333134651184, "rewards/proximity_reward/std": 0.06663305312395096, "reward": 0.4177083373069763, "reward_std": 0.5021805167198181, "frac_reward_zero_std": 0.5, "sampling/sampling_logp_difference/mean": 0.006294678431004286, "sampling/sampling_logp_difference/max": 0.4999983310699463, "sampling/importance_sampling_ratio/min": 0.757157564163208, "sampling/importance_sampling_ratio/mean": 0.9581866264343262, "sampling/importance_sampling_ratio/max": 1.058418869972229, "kl": 0.0006096140714362264, "entropy": 0.06097259745001793, "clip_ratio/low_mean": 0.003105590119957924, "clip_ratio/low_min": 0.003105590119957924, "clip_ratio/high_mean": 0.001552795059978962, "clip_ratio/high_max": 0.001552795059978962, "clip_ratio/region_mean": 0.004658385179936886, "step_time": 0.6864994640000077, "epoch": 0.84375}
28
+ {"step": 28, "loss": -0.0799, "grad_norm": 3.242023468017578, "learning_rate": 2.620263034814632e-07, "num_tokens": 97259.0, "completions/mean_length": 21.03125, "completions/min_length": 18.0, "completions/max_length": 22.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 21.03125, "completions/min_terminated_length": 18.0, "completions/max_terminated_length": 22.0, "rewards/countdown_reward/mean": 0.03125, "rewards/countdown_reward/std": 0.1767766922712326, "rewards/format_reward/mean": 0.078125, "rewards/format_reward/std": 0.04200134426355362, "rewards/valid_numbers_reward/mean": 0.15625, "rewards/valid_numbers_reward/std": 0.08400268852710724, "rewards/proximity_reward/mean": 0.04020833596587181, "rewards/proximity_reward/std": 0.040338512510061264, "reward": 0.3058333396911621, "reward_std": 0.2621923089027405, "frac_reward_zero_std": 0.0, "sampling/sampling_logp_difference/mean": 0.004645894281566143, "sampling/sampling_logp_difference/max": 0.49219369888305664, "sampling/importance_sampling_ratio/min": 0.7788451910018921, "sampling/importance_sampling_ratio/mean": 1.0141183137893677, "sampling/importance_sampling_ratio/max": 1.5722929239273071, "kl": 0.0011130323400720954, "entropy": 0.054960351437330246, "clip_ratio/low_mean": 0.001485884073190391, "clip_ratio/low_min": 0.001485884073190391, "clip_ratio/high_mean": 0.001485884073190391, "clip_ratio/high_max": 0.001485884073190391, "clip_ratio/region_mean": 0.002971768146380782, "step_time": 0.691043074000163, "epoch": 0.875}
29
+ {"step": 29, "loss": -0.1248, "grad_norm": 2.3634800910949707, "learning_rate": 2.2652592093878665e-07, "num_tokens": 100809.0, "completions/mean_length": 22.4375, "completions/min_length": 18.0, "completions/max_length": 25.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 22.4375, "completions/min_terminated_length": 18.0, "completions/max_terminated_length": 25.0, "rewards/countdown_reward/mean": 0.28125, "rewards/countdown_reward/std": 0.45680341124534607, "rewards/format_reward/mean": 0.08124999701976776, "rewards/format_reward/std": 0.03965577483177185, "rewards/valid_numbers_reward/mean": 0.16249999403953552, "rewards/valid_numbers_reward/std": 0.0793115496635437, "rewards/proximity_reward/mean": 0.06205357238650322, "rewards/proximity_reward/std": 0.08783420920372009, "reward": 0.5870535373687744, "reward_std": 0.5920328497886658, "frac_reward_zero_std": 0.0, "sampling/sampling_logp_difference/mean": 0.007978800684213638, "sampling/sampling_logp_difference/max": 0.9378328323364258, "sampling/importance_sampling_ratio/min": 0.2862122058868408, "sampling/importance_sampling_ratio/mean": 1.0282001495361328, "sampling/importance_sampling_ratio/max": 1.4644712209701538, "kl": 0.0005595331895165145, "entropy": 0.03826366737484932, "clip_ratio/low_mean": 0.002785515272989869, "clip_ratio/low_min": 0.002785515272989869, "clip_ratio/high_mean": 0.0013927576364949346, "clip_ratio/high_max": 0.0013927576364949346, "clip_ratio/region_mean": 0.0041782730259001255, "step_time": 0.7093623440000556, "epoch": 0.90625}
30
+ {"step": 30, "loss": 0.0, "grad_norm": 0.005207752343267202, "learning_rate": 1.9289364365516607e-07, "num_tokens": 104319.0, "completions/mean_length": 20.6875, "completions/min_length": 18.0, "completions/max_length": 24.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 20.6875, "completions/min_terminated_length": 18.0, "completions/max_terminated_length": 24.0, "rewards/countdown_reward/mean": 0.5, "rewards/countdown_reward/std": 0.5080004930496216, "rewards/format_reward/mean": 0.10000000149011612, "rewards/format_reward/std": 0.0, "rewards/valid_numbers_reward/mean": 0.20000000298023224, "rewards/valid_numbers_reward/std": 0.0, "rewards/proximity_reward/mean": 0.12000000476837158, "rewards/proximity_reward/std": 0.08128008246421814, "reward": 0.9200000166893005, "reward_std": 0.5892805457115173, "frac_reward_zero_std": 1.0, "sampling/sampling_logp_difference/mean": 0.001935314736329019, "sampling/sampling_logp_difference/max": 0.2494516372680664, "sampling/importance_sampling_ratio/min": 0.879340648651123, "sampling/importance_sampling_ratio/mean": 0.9897561073303223, "sampling/importance_sampling_ratio/max": 1.3560521602630615, "kl": 0.00011867708235513419, "entropy": 0.017748219892382622, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/high_max": 0.0, "clip_ratio/region_mean": 0.0, "step_time": 0.6998922230000062, "epoch": 0.9375}
31
+ {"step": 31, "loss": 0.0611, "grad_norm": 2.1904594898223877, "learning_rate": 1.6135921418712955e-07, "num_tokens": 107724.0, "completions/mean_length": 19.40625, "completions/min_length": 18.0, "completions/max_length": 23.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 19.40625, "completions/min_terminated_length": 18.0, "completions/max_terminated_length": 23.0, "rewards/countdown_reward/mean": 0.15625, "rewards/countdown_reward/std": 0.3689020276069641, "rewards/format_reward/mean": 0.09062500298023224, "rewards/format_reward/std": 0.029614459723234177, "rewards/valid_numbers_reward/mean": 0.16875001788139343, "rewards/valid_numbers_reward/std": 0.07378040999174118, "rewards/proximity_reward/mean": 0.07500000298023224, "rewards/proximity_reward/std": 0.061054062098264694, "reward": 0.4906249940395355, "reward_std": 0.4572972357273102, "frac_reward_zero_std": 0.0, "sampling/sampling_logp_difference/mean": 0.015460848808288574, "sampling/sampling_logp_difference/max": 0.8136651515960693, "sampling/importance_sampling_ratio/min": 0.46403998136520386, "sampling/importance_sampling_ratio/mean": 0.9687584638595581, "sampling/importance_sampling_ratio/max": 1.4587351083755493, "kl": 0.0015718439826741815, "entropy": 0.08870946615934372, "clip_ratio/low_mean": 0.004830917809158564, "clip_ratio/low_min": 0.004830917809158564, "clip_ratio/high_mean": 0.011272141709923744, "clip_ratio/high_max": 0.011272141709923744, "clip_ratio/region_mean": 0.01610305905342102, "step_time": 0.6862520940001104, "epoch": 0.96875}
32
+ {"step": 32, "loss": 0.0706, "grad_norm": 2.8171820640563965, "learning_rate": 1.321380446634342e-07, "num_tokens": 111240.0, "completions/mean_length": 21.875, "completions/min_length": 18.0, "completions/max_length": 23.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 21.875, "completions/min_terminated_length": 18.0, "completions/max_terminated_length": 23.0, "rewards/countdown_reward/mean": 0.0, "rewards/countdown_reward/std": 0.0, "rewards/format_reward/mean": 0.08749999850988388, "rewards/format_reward/std": 0.033601075410842896, "rewards/valid_numbers_reward/mean": 0.13750000298023224, "rewards/valid_numbers_reward/std": 0.09418581426143646, "rewards/proximity_reward/mean": 0.0454629622399807, "rewards/proximity_reward/std": 0.04640965536236763, "reward": 0.2704629600048065, "reward_std": 0.15080341696739197, "frac_reward_zero_std": 0.0, "sampling/sampling_logp_difference/mean": 0.009648799896240234, "sampling/sampling_logp_difference/max": 1.194356918334961, "sampling/importance_sampling_ratio/min": 0.29941776394844055, "sampling/importance_sampling_ratio/mean": 0.9770330190658569, "sampling/importance_sampling_ratio/max": 1.589633822441101, "kl": 0.0017971686320379376, "entropy": 0.059439048171043396, "clip_ratio/low_mean": 0.004285714123398066, "clip_ratio/low_min": 0.004285714123398066, "clip_ratio/high_mean": 0.0028571428265422583, "clip_ratio/high_max": 0.0028571428265422583, "clip_ratio/region_mean": 0.0071428571827709675, "step_time": 0.6916781440002069, "epoch": 1.0}
33
+ {"step": 33, "loss": 0.22, "grad_norm": 5.006729602813721, "learning_rate": 1.0542974530180327e-07, "num_tokens": 114709.0, "completions/mean_length": 20.40625, "completions/min_length": 18.0, "completions/max_length": 23.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 20.40625, "completions/min_terminated_length": 18.0, "completions/max_terminated_length": 23.0, "rewards/countdown_reward/mean": 0.0, "rewards/countdown_reward/std": 0.0, "rewards/format_reward/mean": 0.05937500298023224, "rewards/format_reward/std": 0.04989909008145332, "rewards/valid_numbers_reward/mean": 0.10625000298023224, "rewards/valid_numbers_reward/std": 0.10140147060155869, "rewards/proximity_reward/mean": 0.02578124962747097, "rewards/proximity_reward/std": 0.03299827501177788, "reward": 0.19140625, "reward_std": 0.1729872077703476, "frac_reward_zero_std": 0.0, "sampling/sampling_logp_difference/mean": 0.022636547684669495, "sampling/sampling_logp_difference/max": 1.0278531312942505, "sampling/importance_sampling_ratio/min": 0.17523153126239777, "sampling/importance_sampling_ratio/mean": 0.909177303314209, "sampling/importance_sampling_ratio/max": 1.5272537469863892, "kl": 0.0036370805464684963, "entropy": 0.07416604459285736, "clip_ratio/low_mean": 0.0015313936164602637, "clip_ratio/low_min": 0.0015313936164602637, "clip_ratio/high_mean": 0.016845328733325005, "clip_ratio/high_max": 0.016845328733325005, "clip_ratio/region_mean": 0.018376722931861877, "step_time": 0.6983875639998587, "epoch": 1.03125}
34
+ {"step": 34, "loss": 0.1097, "grad_norm": 5.1096696853637695, "learning_rate": 8.141676086873573e-08, "num_tokens": 118175.0, "completions/mean_length": 20.8125, "completions/min_length": 18.0, "completions/max_length": 28.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 20.8125, "completions/min_terminated_length": 18.0, "completions/max_terminated_length": 28.0, "rewards/countdown_reward/mean": 0.21875, "rewards/countdown_reward/std": 0.420013427734375, "rewards/format_reward/mean": 0.08750000596046448, "rewards/format_reward/std": 0.033601075410842896, "rewards/valid_numbers_reward/mean": 0.16875000298023224, "rewards/valid_numbers_reward/std": 0.07378040999174118, "rewards/proximity_reward/mean": 0.07609374821186066, "rewards/proximity_reward/std": 0.06935650110244751, "reward": 0.5510936975479126, "reward_std": 0.5241543054580688, "frac_reward_zero_std": 0.0, "sampling/sampling_logp_difference/mean": 0.01665877178311348, "sampling/sampling_logp_difference/max": 0.52751624584198, "sampling/importance_sampling_ratio/min": 0.5610782504081726, "sampling/importance_sampling_ratio/mean": 0.935814380645752, "sampling/importance_sampling_ratio/max": 1.783529281616211, "kl": 0.0037688622251152992, "entropy": 0.11135981231927872, "clip_ratio/low_mean": 0.0030030030757188797, "clip_ratio/low_min": 0.0030030030757188797, "clip_ratio/high_mean": 0.006006006151437759, "clip_ratio/high_max": 0.006006006151437759, "clip_ratio/region_mean": 0.009009009227156639, "step_time": 0.7239749829998345, "epoch": 1.0625}
35
+ {"step": 35, "loss": 0.0727, "grad_norm": 3.130201816558838, "learning_rate": 6.026312439675551e-08, "num_tokens": 121518.0, "completions/mean_length": 16.96875, "completions/min_length": 12.0, "completions/max_length": 22.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 16.96875, "completions/min_terminated_length": 12.0, "completions/max_terminated_length": 22.0, "rewards/countdown_reward/mean": 0.34375, "rewards/countdown_reward/std": 0.4825586974620819, "rewards/format_reward/mean": 0.09062500298023224, "rewards/format_reward/std": 0.029614459723234177, "rewards/valid_numbers_reward/mean": 0.08750000596046448, "rewards/valid_numbers_reward/std": 0.10080322623252869, "rewards/proximity_reward/mean": 0.07172618806362152, "rewards/proximity_reward/std": 0.09507618099451065, "reward": 0.5936011672019958, "reward_std": 0.6709741950035095, "frac_reward_zero_std": 0.0, "sampling/sampling_logp_difference/mean": 0.008752727881073952, "sampling/sampling_logp_difference/max": 0.7403793334960938, "sampling/importance_sampling_ratio/min": 0.6212528347969055, "sampling/importance_sampling_ratio/mean": 1.0003535747528076, "sampling/importance_sampling_ratio/max": 1.7464821338653564, "kl": 0.0008046461152844131, "entropy": 0.04473040997982025, "clip_ratio/low_mean": 0.003683241317048669, "clip_ratio/low_min": 0.003683241317048669, "clip_ratio/high_mean": 0.0018416206585243344, "clip_ratio/high_max": 0.0018416206585243344, "clip_ratio/region_mean": 0.005524862091988325, "step_time": 0.6803748840000026, "epoch": 1.09375}
36
+ {"step": 36, "loss": 0.017, "grad_norm": 1.5974429845809937, "learning_rate": 4.2113336672471245e-08, "num_tokens": 124944.0, "completions/mean_length": 18.5625, "completions/min_length": 12.0, "completions/max_length": 24.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 18.5625, "completions/min_terminated_length": 12.0, "completions/max_terminated_length": 24.0, "rewards/countdown_reward/mean": 0.5, "rewards/countdown_reward/std": 0.5080004930496216, "rewards/format_reward/mean": 0.09062500298023224, "rewards/format_reward/std": 0.029614459723234177, "rewards/valid_numbers_reward/mean": 0.10625000298023224, "rewards/valid_numbers_reward/std": 0.10140147060155869, "rewards/proximity_reward/mean": 0.1026785746216774, "rewards/proximity_reward/std": 0.09999588131904602, "reward": 0.7995535731315613, "reward_std": 0.7141795754432678, "frac_reward_zero_std": 0.5, "sampling/sampling_logp_difference/mean": 0.008274354040622711, "sampling/sampling_logp_difference/max": 0.4873225688934326, "sampling/importance_sampling_ratio/min": 0.6722076535224915, "sampling/importance_sampling_ratio/mean": 0.9889246821403503, "sampling/importance_sampling_ratio/max": 1.6599711179733276, "kl": 0.00030721016810275614, "entropy": 0.0578792467713356, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/high_mean": 0.001683501643128693, "clip_ratio/high_max": 0.001683501643128693, "clip_ratio/region_mean": 0.001683501643128693, "step_time": 0.6986799229998724, "epoch": 1.125}
37
+ {"step": 37, "loss": 0.0001, "grad_norm": 0.052407167851924896, "learning_rate": 2.7091379149682682e-08, "num_tokens": 128513.0, "completions/mean_length": 23.53125, "completions/min_length": 22.0, "completions/max_length": 29.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 23.53125, "completions/min_terminated_length": 22.0, "completions/max_terminated_length": 29.0, "rewards/countdown_reward/mean": 0.0, "rewards/countdown_reward/std": 0.0, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/valid_numbers_reward/mean": 0.0, "rewards/valid_numbers_reward/std": 0.0, "rewards/proximity_reward/mean": 0.0, "rewards/proximity_reward/std": 0.0, "reward": 0.0, "reward_std": 0.0, "frac_reward_zero_std": 1.0, "sampling/sampling_logp_difference/mean": 0.007070348598062992, "sampling/sampling_logp_difference/max": 1.1611120700836182, "sampling/importance_sampling_ratio/min": 0.3940061330795288, "sampling/importance_sampling_ratio/mean": 1.020906686782837, "sampling/importance_sampling_ratio/max": 1.3242827653884888, "kl": 0.0013774223625659943, "entropy": 0.030706292018294334, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/high_max": 0.0, "clip_ratio/region_mean": 0.0, "step_time": 0.7337732739999865, "epoch": 1.15625}
38
+ {"step": 38, "loss": -0.0881, "grad_norm": 1.7202657461166382, "learning_rate": 1.5299867030334813e-08, "num_tokens": 131803.0, "completions/mean_length": 15.8125, "completions/min_length": 12.0, "completions/max_length": 19.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 15.8125, "completions/min_terminated_length": 12.0, "completions/max_terminated_length": 19.0, "rewards/countdown_reward/mean": 0.0, "rewards/countdown_reward/std": 0.0, "rewards/format_reward/mean": 0.10000000149011612, "rewards/format_reward/std": 0.0, "rewards/valid_numbers_reward/mean": 0.125, "rewards/valid_numbers_reward/std": 0.09837387502193451, "rewards/proximity_reward/mean": 0.040744051337242126, "rewards/proximity_reward/std": 0.03569040447473526, "reward": 0.26574403047561646, "reward_std": 0.13137724995613098, "frac_reward_zero_std": 0.0, "sampling/sampling_logp_difference/mean": 0.00469952542334795, "sampling/sampling_logp_difference/max": 0.21667885780334473, "sampling/importance_sampling_ratio/min": 0.6875080466270447, "sampling/importance_sampling_ratio/mean": 1.0013912916183472, "sampling/importance_sampling_ratio/max": 1.2280179262161255, "kl": 0.00043047123472206295, "entropy": 0.05823312699794769, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/high_mean": 0.013833992183208466, "clip_ratio/high_max": 0.013833992183208466, "clip_ratio/region_mean": 0.013833992183208466, "step_time": 0.6611508340001819, "epoch": 1.1875}
39
+ {"step": 39, "loss": -0.0592, "grad_norm": 1.1424243450164795, "learning_rate": 6.819348298638839e-09, "num_tokens": 135334.0, "completions/mean_length": 20.84375, "completions/min_length": 18.0, "completions/max_length": 26.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 20.84375, "completions/min_terminated_length": 18.0, "completions/max_terminated_length": 26.0, "rewards/countdown_reward/mean": 0.96875, "rewards/countdown_reward/std": 0.1767766922712326, "rewards/format_reward/mean": 0.10000000149011612, "rewards/format_reward/std": 0.0, "rewards/valid_numbers_reward/mean": 0.19374999403953552, "rewards/valid_numbers_reward/std": 0.0353553406894207, "rewards/proximity_reward/mean": 0.19374999403953552, "rewards/proximity_reward/std": 0.0353553406894207, "reward": 1.4562499523162842, "reward_std": 0.2474873661994934, "frac_reward_zero_std": 0.5, "sampling/sampling_logp_difference/mean": 0.00784258358180523, "sampling/sampling_logp_difference/max": 0.9508624076843262, "sampling/importance_sampling_ratio/min": 0.4590398967266083, "sampling/importance_sampling_ratio/mean": 1.0131592750549316, "sampling/importance_sampling_ratio/max": 2.6269302368164062, "kl": 0.0019472006242722273, "entropy": 0.05225186049938202, "clip_ratio/low_mean": 0.0014992504147812724, "clip_ratio/low_min": 0.0014992504147812724, "clip_ratio/high_mean": 0.00599700165912509, "clip_ratio/high_max": 0.00599700165912509, "clip_ratio/region_mean": 0.007496251724660397, "step_time": 0.716618513999947, "epoch": 1.21875}
40
+ {"step": 40, "loss": -0.0239, "grad_norm": 2.2067079544067383, "learning_rate": 1.7077534966650765e-09, "num_tokens": 138807.0, "completions/mean_length": 20.53125, "completions/min_length": 18.0, "completions/max_length": 23.0, "completions/clipped_ratio": 0.0, "completions/mean_terminated_length": 20.53125, "completions/min_terminated_length": 18.0, "completions/max_terminated_length": 23.0, "rewards/countdown_reward/mean": 0.625, "rewards/countdown_reward/std": 0.49186936020851135, "rewards/format_reward/mean": 0.10000000149011612, "rewards/format_reward/std": 0.0, "rewards/valid_numbers_reward/mean": 0.17500001192092896, "rewards/valid_numbers_reward/std": 0.06720215082168579, "rewards/proximity_reward/mean": 0.1411931812763214, "rewards/proximity_reward/std": 0.08057914674282074, "reward": 1.0411932468414307, "reward_std": 0.6069347858428955, "frac_reward_zero_std": 0.5, "sampling/sampling_logp_difference/mean": 0.010136268101632595, "sampling/sampling_logp_difference/max": 0.7177395820617676, "sampling/importance_sampling_ratio/min": 0.6189467310905457, "sampling/importance_sampling_ratio/mean": 0.9915159940719604, "sampling/importance_sampling_ratio/max": 1.6101794242858887, "kl": 0.0008815153851173818, "entropy": 0.06015819311141968, "clip_ratio/low_mean": 0.006088280119001865, "clip_ratio/low_min": 0.006088280119001865, "clip_ratio/high_mean": 0.0015220700297504663, "clip_ratio/high_max": 0.0015220700297504663, "clip_ratio/region_mean": 0.007610349915921688, "step_time": 0.6921010940000087, "epoch": 1.25}
41
+ {"step": 40, "train_runtime": 37.5295, "train_samples_per_second": 34.106, "train_steps_per_second": 1.066, "total_flos": 0.0, "train_loss": 0.00969338510142279, "epoch": 1.25}
runs/smoke_staged.log ADDED
@@ -0,0 +1,211 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ === smoke_staged: stage 0 ===
3
+
4
+
5
+
6
+
7
+
8
+ 2026-06-21 11:26:03,914 - INFO - autotuner.py:256 - flashinfer.jit: [Autotuner]: Autotuning process starts ...
9
+ 2026-06-21 11:26:03,920 - INFO - autotuner.py:262 - flashinfer.jit: [Autotuner]: Autotuning process ends
10
+
11
+
12
+ wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
13
+ wandb: Currently logged in as: kishanvavdara to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
14
+ wandb: Tracking run with wandb version 0.27.2
15
+ wandb: Run data is saved locally in /content/llm-zero-lite/wandb/run-20260621_112606-xhfzel3m
16
+ wandb: Run `wandb offline` to turn off syncing.
17
+ wandb: Syncing run smoke_staged_stage_0
18
+ wandb: ⭐️ View project at https://wandb.ai/kishanvavdara/llm-zero-lite-smoke
19
+ wandb: 🚀 View run at https://wandb.ai/kishanvavdara/llm-zero-lite-smoke/runs/xhfzel3m
20
+ wandb: Detected [openai] in use.
21
+ wandb: Use W&B Weave for improved LLM call tracing. Install Weave with `pip install weave` then add `import weave` to the top of your script.
22
+ wandb: For more information, check out the docs at: https://weave-docs.wandb.ai
23
+ {'loss': -0.1839, 'grad_norm': 5.719128608703613, 'learning_rate': 0.0, 'num_tokens': 3412.0, 'completions/mean_length': 18.875, 'completions/min_length': 12.0, 'completions/max_length': 24.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 18.875, 'completions/min_terminated_length': 12.0, 'completions/max_terminated_length': 24.0, 'rewards/countdown_reward/mean': 0.3125, 'rewards/countdown_reward/std': 0.4709290862083435, 'rewards/format_reward/mean': 0.10000000149011612, 'rewards/format_reward/std': 0.0, 'rewards/valid_numbers_reward/mean': 0.17499999701976776, 'rewards/valid_numbers_reward/std': 0.06720215082168579, 'rewards/proximity_reward/mean': 0.10062500089406967, 'rewards/proximity_reward/std': 0.07458577305078506, 'reward': 0.6881250143051147, 'reward_std': 0.5632839798927307, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.014817753806710243, 'sampling/sampling_logp_difference/max': 0.6194897890090942, 'sampling/importance_sampling_ratio/min': 0.3401852548122406, 'sampling/importance_sampling_ratio/mean': 1.0280770063400269, 'sampling/importance_sampling_ratio/max': 1.5959720611572266, 'kl': 0.0, 'entropy': 0.09279054403305054, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 3.743490957999711, 'epoch': 0.0625}
24
+ {'loss': 0.0101, 'grad_norm': 2.885303258895874, 'learning_rate': 1e-06, 'num_tokens': 6947.0, 'completions/mean_length': 22.21875, 'completions/min_length': 18.0, 'completions/max_length': 24.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 22.21875, 'completions/min_terminated_length': 18.0, 'completions/max_terminated_length': 24.0, 'rewards/countdown_reward/mean': 0.625, 'rewards/countdown_reward/std': 0.49186936020851135, 'rewards/format_reward/mean': 0.07500000298023224, 'rewards/format_reward/std': 0.04399413242936134, 'rewards/valid_numbers_reward/mean': 0.15000000596046448, 'rewards/valid_numbers_reward/std': 0.08798826485872269, 'rewards/proximity_reward/mean': 0.12783482670783997, 'rewards/proximity_reward/std': 0.0949597954750061, 'reward': 0.9778348207473755, 'reward_std': 0.691416323184967, 'frac_reward_zero_std': 0.5, 'sampling/sampling_logp_difference/mean': 0.011655616573989391, 'sampling/sampling_logp_difference/max': 0.4029996395111084, 'sampling/importance_sampling_ratio/min': 0.48065969347953796, 'sampling/importance_sampling_ratio/mean': 0.9353974461555481, 'sampling/importance_sampling_ratio/max': 1.7168827056884766, 'kl': 0.0, 'entropy': 0.06589090079069138, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 0.7587055470003179, 'epoch': 0.125}
25
+ {'loss': -0.0014, 'grad_norm': 2.2229199409484863, 'learning_rate': 9.931806517013612e-07, 'num_tokens': 10428.0, 'completions/mean_length': 20.53125, 'completions/min_length': 18.0, 'completions/max_length': 24.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 20.53125, 'completions/min_terminated_length': 18.0, 'completions/max_terminated_length': 24.0, 'rewards/countdown_reward/mean': 0.25, 'rewards/countdown_reward/std': 0.4399413466453552, 'rewards/format_reward/mean': 0.08749999850988388, 'rewards/format_reward/std': 0.033601075410842896, 'rewards/valid_numbers_reward/mean': 0.17499999701976776, 'rewards/valid_numbers_reward/std': 0.06720215082168579, 'rewards/proximity_reward/mean': 0.0885416716337204, 'rewards/proximity_reward/std': 0.07646888494491577, 'reward': 0.6010416746139526, 'reward_std': 0.5415659546852112, 'frac_reward_zero_std': 0.5, 'sampling/sampling_logp_difference/mean': 0.018785474821925163, 'sampling/sampling_logp_difference/max': 1.0769587755203247, 'sampling/importance_sampling_ratio/min': 0.2317689210176468, 'sampling/importance_sampling_ratio/mean': 0.9618514776229858, 'sampling/importance_sampling_ratio/max': 1.5981510877609253, 'kl': 0.003546408610418439, 'entropy': 0.07333880662918091, 'clip_ratio/low_mean': 0.0030441400595009327, 'clip_ratio/low_min': 0.0030441400595009327, 'clip_ratio/high_mean': 0.00913241971284151, 'clip_ratio/high_max': 0.00913241971284151, 'clip_ratio/region_mean': 0.01217656023800373, 'step_time': 0.72251101799975, 'epoch': 0.1875}
26
+ {'loss': 0.0735, 'grad_norm': 2.2820451259613037, 'learning_rate': 9.729086208503173e-07, 'num_tokens': 13915.0, 'completions/mean_length': 20.71875, 'completions/min_length': 18.0, 'completions/max_length': 26.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 20.71875, 'completions/min_terminated_length': 18.0, 'completions/max_terminated_length': 26.0, 'rewards/countdown_reward/mean': 0.125, 'rewards/countdown_reward/std': 0.33601075410842896, 'rewards/format_reward/mean': 0.09062500298023224, 'rewards/format_reward/std': 0.029614459723234177, 'rewards/valid_numbers_reward/mean': 0.16875000298023224, 'rewards/valid_numbers_reward/std': 0.07378040999174118, 'rewards/proximity_reward/mean': 0.05255208909511566, 'rewards/proximity_reward/std': 0.0620746947824955, 'reward': 0.43692708015441895, 'reward_std': 0.4234214723110199, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.012193999253213406, 'sampling/sampling_logp_difference/max': 0.773352861404419, 'sampling/importance_sampling_ratio/min': 0.5003921389579773, 'sampling/importance_sampling_ratio/mean': 0.9831472039222717, 'sampling/importance_sampling_ratio/max': 1.6385778188705444, 'kl': 0.0016355457482859492, 'entropy': 0.07203121483325958, 'clip_ratio/low_mean': 0.004524887073785067, 'clip_ratio/low_min': 0.004524887073785067, 'clip_ratio/high_mean': 0.004524887073785067, 'clip_ratio/high_max': 0.004524887073785067, 'clip_ratio/region_mean': 0.009049774147570133, 'step_time': 0.7308499579994532, 'epoch': 0.25}
27
+ {'loss': -0.0733, 'grad_norm': 2.4893078804016113, 'learning_rate': 9.397368756032444e-07, 'num_tokens': 17416.0, 'completions/mean_length': 20.65625, 'completions/min_length': 18.0, 'completions/max_length': 23.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 20.65625, 'completions/min_terminated_length': 18.0, 'completions/max_terminated_length': 23.0, 'rewards/countdown_reward/mean': 0.5625, 'rewards/countdown_reward/std': 0.504016101360321, 'rewards/format_reward/mean': 0.07500000298023224, 'rewards/format_reward/std': 0.04399413242936134, 'rewards/valid_numbers_reward/mean': 0.15000000596046448, 'rewards/valid_numbers_reward/std': 0.08798826485872269, 'rewards/proximity_reward/mean': 0.12111110985279083, 'rewards/proximity_reward/std': 0.09242431819438934, 'reward': 0.9086111187934875, 'reward_std': 0.6909820437431335, 'frac_reward_zero_std': 0.75, 'sampling/sampling_logp_difference/mean': 0.0068531762808561325, 'sampling/sampling_logp_difference/max': 0.5645411014556885, 'sampling/importance_sampling_ratio/min': 0.5731061100959778, 'sampling/importance_sampling_ratio/mean': 1.0513486862182617, 'sampling/importance_sampling_ratio/max': 2.3135290145874023, 'kl': 0.0012599638430401683, 'entropy': 0.047102976590394974, 'clip_ratio/low_mean': 0.0015128592494875193, 'clip_ratio/low_min': 0.0015128592494875193, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0015128592494875193, 'step_time': 0.710334069000055, 'epoch': 0.3125}
28
+ {'loss': 0.113, 'grad_norm': 2.6415159702301025, 'learning_rate': 8.945702546981968e-07, 'num_tokens': 20853.0, 'completions/mean_length': 18.90625, 'completions/min_length': 12.0, 'completions/max_length': 24.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 18.90625, 'completions/min_terminated_length': 12.0, 'completions/max_terminated_length': 24.0, 'rewards/countdown_reward/mean': 0.28125, 'rewards/countdown_reward/std': 0.45680341124534607, 'rewards/format_reward/mean': 0.09687499701976776, 'rewards/format_reward/std': 0.01767767034471035, 'rewards/valid_numbers_reward/mean': 0.13750000298023224, 'rewards/valid_numbers_reward/std': 0.09418581426143646, 'rewards/proximity_reward/mean': 0.07734374701976776, 'rewards/proximity_reward/std': 0.0844610333442688, 'reward': 0.592968761920929, 'reward_std': 0.5879430174827576, 'frac_reward_zero_std': 0.25, 'sampling/sampling_logp_difference/mean': 0.011994819156825542, 'sampling/sampling_logp_difference/max': 0.5250324010848999, 'sampling/importance_sampling_ratio/min': 0.678278923034668, 'sampling/importance_sampling_ratio/mean': 1.0119421482086182, 'sampling/importance_sampling_ratio/max': 1.6314356327056885, 'kl': 0.0014897827059030533, 'entropy': 0.07113193720579147, 'clip_ratio/low_mean': 0.0033057851251214743, 'clip_ratio/low_min': 0.0033057851251214743, 'clip_ratio/high_mean': 0.0033057851251214743, 'clip_ratio/high_max': 0.0033057851251214743, 'clip_ratio/region_mean': 0.0066115702502429485, 'step_time': 0.7184356580000895, 'epoch': 0.375}
29
+ {'loss': 0.0907, 'grad_norm': 1.911924958229065, 'learning_rate': 8.386407858128706e-07, 'num_tokens': 24377.0, 'completions/mean_length': 22.125, 'completions/min_length': 18.0, 'completions/max_length': 29.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 22.125, 'completions/min_terminated_length': 18.0, 'completions/max_terminated_length': 29.0, 'rewards/countdown_reward/mean': 0.25, 'rewards/countdown_reward/std': 0.4399413466453552, 'rewards/format_reward/mean': 0.05937500298023224, 'rewards/format_reward/std': 0.04989909008145332, 'rewards/valid_numbers_reward/mean': 0.11875000596046448, 'rewards/valid_numbers_reward/std': 0.09979818016290665, 'rewards/proximity_reward/mean': 0.05595213174819946, 'rewards/proximity_reward/std': 0.08524730801582336, 'reward': 0.4840771555900574, 'reward_std': 0.6120077967643738, 'frac_reward_zero_std': 0.5, 'sampling/sampling_logp_difference/mean': 0.008370033465325832, 'sampling/sampling_logp_difference/max': 0.4872523546218872, 'sampling/importance_sampling_ratio/min': 0.7153430581092834, 'sampling/importance_sampling_ratio/mean': 1.0180648565292358, 'sampling/importance_sampling_ratio/max': 1.6259102821350098, 'kl': 0.000581703963689506, 'entropy': 0.044528089463710785, 'clip_ratio/low_mean': 0.0014124293811619282, 'clip_ratio/low_min': 0.0014124293811619282, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0014124293811619282, 'step_time': 0.7480844470001102, 'epoch': 0.4375}
30
+ {'loss': 0.0198, 'grad_norm': 2.209592819213867, 'learning_rate': 7.734740790612136e-07, 'num_tokens': 27823.0, 'completions/mean_length': 20.4375, 'completions/min_length': 18.0, 'completions/max_length': 25.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 20.4375, 'completions/min_terminated_length': 18.0, 'completions/max_terminated_length': 25.0, 'rewards/countdown_reward/mean': 0.125, 'rewards/countdown_reward/std': 0.33601075410842896, 'rewards/format_reward/mean': 0.08124999701976776, 'rewards/format_reward/std': 0.03965577483177185, 'rewards/valid_numbers_reward/mean': 0.10625000298023224, 'rewards/valid_numbers_reward/std': 0.10140147060155869, 'rewards/proximity_reward/mean': 0.046341296285390854, 'rewards/proximity_reward/std': 0.06670770794153214, 'reward': 0.35884130001068115, 'reward_std': 0.46127012372016907, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.01414323691278696, 'sampling/sampling_logp_difference/max': 0.9508602619171143, 'sampling/importance_sampling_ratio/min': 0.41342437267303467, 'sampling/importance_sampling_ratio/mean': 1.0203397274017334, 'sampling/importance_sampling_ratio/max': 1.6892083883285522, 'kl': 0.0015636704629287124, 'entropy': 0.08313256502151489, 'clip_ratio/low_mean': 0.004587155766785145, 'clip_ratio/low_min': 0.004587155766785145, 'clip_ratio/high_mean': 0.0030581040773540735, 'clip_ratio/high_max': 0.0030581040773540735, 'clip_ratio/region_mean': 0.007645260076969862, 'step_time': 0.7207483080001111, 'epoch': 0.5}
31
+ {'loss': 0.0158, 'grad_norm': 1.6458412408828735, 'learning_rate': 7.008477123264847e-07, 'num_tokens': 31272.0, 'completions/mean_length': 20.53125, 'completions/min_length': 16.0, 'completions/max_length': 26.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 20.53125, 'completions/min_terminated_length': 16.0, 'completions/max_terminated_length': 26.0, 'rewards/countdown_reward/mean': 0.0, 'rewards/countdown_reward/std': 0.0, 'rewards/format_reward/mean': 0.05937500298023224, 'rewards/format_reward/std': 0.04989909008145332, 'rewards/valid_numbers_reward/mean': 0.11875000596046448, 'rewards/valid_numbers_reward/std': 0.09979818016290665, 'rewards/proximity_reward/mean': 0.03993169963359833, 'rewards/proximity_reward/std': 0.048182517290115356, 'reward': 0.21805670857429504, 'reward_std': 0.1864890307188034, 'frac_reward_zero_std': 0.25, 'sampling/sampling_logp_difference/mean': 0.007480533793568611, 'sampling/sampling_logp_difference/max': 0.33918583393096924, 'sampling/importance_sampling_ratio/min': 0.6571818590164185, 'sampling/importance_sampling_ratio/mean': 0.9944100379943848, 'sampling/importance_sampling_ratio/max': 1.4445140361785889, 'kl': 0.0010829430539160967, 'entropy': 0.06321606040000916, 'clip_ratio/low_mean': 0.0015220700297504663, 'clip_ratio/low_min': 0.0015220700297504663, 'clip_ratio/high_mean': 0.004566209856420755, 'clip_ratio/high_max': 0.004566209856420755, 'clip_ratio/region_mean': 0.006088280119001865, 'step_time': 0.7197023180005999, 'epoch': 0.5625}
32
+ {'loss': -0.0868, 'grad_norm': 2.8024051189422607, 'learning_rate': 6.227427435703995e-07, 'num_tokens': 34710.0, 'completions/mean_length': 19.1875, 'completions/min_length': 12.0, 'completions/max_length': 28.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 19.1875, 'completions/min_terminated_length': 12.0, 'completions/max_terminated_length': 28.0, 'rewards/countdown_reward/mean': 0.28125, 'rewards/countdown_reward/std': 0.45680341124534607, 'rewards/format_reward/mean': 0.08437500149011612, 'rewards/format_reward/std': 0.03689020499587059, 'rewards/valid_numbers_reward/mean': 0.08125000447034836, 'rewards/valid_numbers_reward/std': 0.09979818016290665, 'rewards/proximity_reward/mean': 0.059067800641059875, 'rewards/proximity_reward/std': 0.09065822511911392, 'reward': 0.5059428215026855, 'reward_std': 0.6380220651626587, 'frac_reward_zero_std': 0.25, 'sampling/sampling_logp_difference/mean': 0.014419648796319962, 'sampling/sampling_logp_difference/max': 0.746462345123291, 'sampling/importance_sampling_ratio/min': 0.4375057816505432, 'sampling/importance_sampling_ratio/mean': 0.9968295097351074, 'sampling/importance_sampling_ratio/max': 1.9589970111846924, 'kl': 0.003762712236493826, 'entropy': 0.07867645472288132, 'clip_ratio/low_mean': 0.004885993432253599, 'clip_ratio/low_min': 0.004885993432253599, 'clip_ratio/high_mean': 0.008143322542309761, 'clip_ratio/high_max': 0.008143322542309761, 'clip_ratio/region_mean': 0.013029315508902073, 'step_time': 0.7379443970003194, 'epoch': 0.625}
33
+ {'loss': 0.0, 'grad_norm': 0.005773977376520634, 'learning_rate': 5.412896727361662e-07, 'num_tokens': 38166.0, 'completions/mean_length': 19.75, 'completions/min_length': 12.0, 'completions/max_length': 23.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 19.75, 'completions/min_terminated_length': 12.0, 'completions/max_terminated_length': 23.0, 'rewards/countdown_reward/mean': 0.25, 'rewards/countdown_reward/std': 0.4399413466453552, 'rewards/format_reward/mean': 0.07500000298023224, 'rewards/format_reward/std': 0.04399413242936134, 'rewards/valid_numbers_reward/mean': 0.10000000149011612, 'rewards/valid_numbers_reward/std': 0.10160010308027267, 'rewards/proximity_reward/mean': 0.06000000238418579, 'rewards/proximity_reward/std': 0.08378158509731293, 'reward': 0.48500001430511475, 'reward_std': 0.6084777116775513, 'frac_reward_zero_std': 1.0, 'sampling/sampling_logp_difference/mean': 0.003858300158753991, 'sampling/sampling_logp_difference/max': 0.2500002384185791, 'sampling/importance_sampling_ratio/min': 0.778872013092041, 'sampling/importance_sampling_ratio/mean': 1.0539686679840088, 'sampling/importance_sampling_ratio/max': 1.2945910692214966, 'kl': 6.735674105584621e-05, 'entropy': 0.027388425543904305, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 0.6988498279997657, 'epoch': 0.6875}
34
+ {'loss': 0.0087, 'grad_norm': 1.5385663509368896, 'learning_rate': 4.5871032726383385e-07, 'num_tokens': 41555.0, 'completions/mean_length': 18.40625, 'completions/min_length': 12.0, 'completions/max_length': 23.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 18.40625, 'completions/min_terminated_length': 12.0, 'completions/max_terminated_length': 23.0, 'rewards/countdown_reward/mean': 0.0, 'rewards/countdown_reward/std': 0.0, 'rewards/format_reward/mean': 0.07500000298023224, 'rewards/format_reward/std': 0.04399413242936134, 'rewards/valid_numbers_reward/mean': 0.09375, 'rewards/valid_numbers_reward/std': 0.10140147060155869, 'rewards/proximity_reward/mean': 0.008779370225965977, 'rewards/proximity_reward/std': 0.017096128314733505, 'reward': 0.17752937972545624, 'reward_std': 0.14045517146587372, 'frac_reward_zero_std': 0.5, 'sampling/sampling_logp_difference/mean': 0.007245893124490976, 'sampling/sampling_logp_difference/max': 0.310931921005249, 'sampling/importance_sampling_ratio/min': 0.6769415140151978, 'sampling/importance_sampling_ratio/mean': 0.9727678298950195, 'sampling/importance_sampling_ratio/max': 1.262580156326294, 'kl': 0.0009706447599455714, 'entropy': 0.05887338146567345, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.003395585808902979, 'clip_ratio/high_max': 0.003395585808902979, 'clip_ratio/region_mean': 0.003395585808902979, 'step_time': 0.6943323579998832, 'epoch': 0.75}
35
+ {'loss': 0.0215, 'grad_norm': 1.560404658317566, 'learning_rate': 3.772572564296004e-07, 'num_tokens': 45033.0, 'completions/mean_length': 20.1875, 'completions/min_length': 18.0, 'completions/max_length': 25.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 20.1875, 'completions/min_terminated_length': 18.0, 'completions/max_terminated_length': 25.0, 'rewards/countdown_reward/mean': 0.34375, 'rewards/countdown_reward/std': 0.4825586974620819, 'rewards/format_reward/mean': 0.10000000149011612, 'rewards/format_reward/std': 0.0, 'rewards/valid_numbers_reward/mean': 0.19374999403953552, 'rewards/valid_numbers_reward/std': 0.0353553406894207, 'rewards/proximity_reward/mean': 0.1015625074505806, 'rewards/proximity_reward/std': 0.07403615117073059, 'reward': 0.7390625476837158, 'reward_std': 0.5614259839057922, 'frac_reward_zero_std': 0.5, 'sampling/sampling_logp_difference/mean': 0.007800778839737177, 'sampling/sampling_logp_difference/max': 0.3872795104980469, 'sampling/importance_sampling_ratio/min': 0.5836712121963501, 'sampling/importance_sampling_ratio/mean': 0.9557247161865234, 'sampling/importance_sampling_ratio/max': 1.304115653038025, 'kl': 0.0010749027132987976, 'entropy': 0.0642925500869751, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0030959751456975937, 'clip_ratio/high_max': 0.0030959751456975937, 'clip_ratio/region_mean': 0.0030959751456975937, 'step_time': 0.7162068279994855, 'epoch': 0.8125}
36
+ {'loss': -0.0381, 'grad_norm': 1.8875806331634521, 'learning_rate': 2.9915228767351535e-07, 'num_tokens': 48498.0, 'completions/mean_length': 20.53125, 'completions/min_length': 18.0, 'completions/max_length': 22.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 20.53125, 'completions/min_terminated_length': 18.0, 'completions/max_terminated_length': 22.0, 'rewards/countdown_reward/mean': 0.09375, 'rewards/countdown_reward/std': 0.2961445748806, 'rewards/format_reward/mean': 0.07187500596046448, 'rewards/format_reward/std': 0.04568034037947655, 'rewards/valid_numbers_reward/mean': 0.14375001192092896, 'rewards/valid_numbers_reward/std': 0.0913606807589531, 'rewards/proximity_reward/mean': 0.05458333343267441, 'rewards/proximity_reward/std': 0.05604498088359833, 'reward': 0.36395832896232605, 'reward_std': 0.4044833183288574, 'frac_reward_zero_std': 0.25, 'sampling/sampling_logp_difference/mean': 0.005446118302643299, 'sampling/sampling_logp_difference/max': 0.6850378513336182, 'sampling/importance_sampling_ratio/min': 0.5654681324958801, 'sampling/importance_sampling_ratio/mean': 0.9761245250701904, 'sampling/importance_sampling_ratio/max': 1.5083339214324951, 'kl': 0.00127152178902179, 'entropy': 0.06002233922481537, 'clip_ratio/low_mean': 0.007610349915921688, 'clip_ratio/low_min': 0.007610349915921688, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.007610349915921688, 'step_time': 0.6907684589996279, 'epoch': 0.875}
37
+ {'loss': -0.0388, 'grad_norm': 0.9855704307556152, 'learning_rate': 2.2652592093878665e-07, 'num_tokens': 52022.0, 'completions/mean_length': 21.375, 'completions/min_length': 18.0, 'completions/max_length': 23.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 21.375, 'completions/min_terminated_length': 18.0, 'completions/max_terminated_length': 23.0, 'rewards/countdown_reward/mean': 0.3125, 'rewards/countdown_reward/std': 0.4709290862083435, 'rewards/format_reward/mean': 0.08437500149011612, 'rewards/format_reward/std': 0.03689020499587059, 'rewards/valid_numbers_reward/mean': 0.16875000298023224, 'rewards/valid_numbers_reward/std': 0.07378040999174118, 'rewards/proximity_reward/mean': 0.07532738149166107, 'rewards/proximity_reward/std': 0.08653823286294937, 'reward': 0.6409524083137512, 'reward_std': 0.599563717842102, 'frac_reward_zero_std': 0.75, 'sampling/sampling_logp_difference/mean': 0.006025237031280994, 'sampling/sampling_logp_difference/max': 0.5645420551300049, 'sampling/importance_sampling_ratio/min': 0.6414832472801208, 'sampling/importance_sampling_ratio/mean': 0.9659491181373596, 'sampling/importance_sampling_ratio/max': 1.303356647491455, 'kl': 0.0008252715924754739, 'entropy': 0.025792622938752174, 'clip_ratio/low_mean': 0.001461988314986229, 'clip_ratio/low_min': 0.001461988314986229, 'clip_ratio/high_mean': 0.001461988314986229, 'clip_ratio/high_max': 0.001461988314986229, 'clip_ratio/region_mean': 0.002923976629972458, 'step_time': 0.6958222980001665, 'epoch': 0.9375}
38
+ {'loss': -0.0193, 'grad_norm': 3.257521390914917, 'learning_rate': 1.6135921418712955e-07, 'num_tokens': 55479.0, 'completions/mean_length': 20.53125, 'completions/min_length': 18.0, 'completions/max_length': 23.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 20.53125, 'completions/min_terminated_length': 18.0, 'completions/max_terminated_length': 23.0, 'rewards/countdown_reward/mean': 0.125, 'rewards/countdown_reward/std': 0.33601075410842896, 'rewards/format_reward/mean': 0.08437500894069672, 'rewards/format_reward/std': 0.03689020499587059, 'rewards/valid_numbers_reward/mean': 0.14375001192092896, 'rewards/valid_numbers_reward/std': 0.0913606807589531, 'rewards/proximity_reward/mean': 0.0591898150742054, 'rewards/proximity_reward/std': 0.06511926651000977, 'reward': 0.41231483221054077, 'reward_std': 0.4417669177055359, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.009429214522242546, 'sampling/sampling_logp_difference/max': 0.4696979522705078, 'sampling/importance_sampling_ratio/min': 0.5590547323226929, 'sampling/importance_sampling_ratio/mean': 0.9672008752822876, 'sampling/importance_sampling_ratio/max': 1.4337674379348755, 'kl': 0.0014836424961686134, 'entropy': 0.079550601541996, 'clip_ratio/low_mean': 0.004566209856420755, 'clip_ratio/low_min': 0.004566209856420755, 'clip_ratio/high_mean': 0.0030441400595009327, 'clip_ratio/high_max': 0.0030441400595009327, 'clip_ratio/region_mean': 0.007610349915921688, 'step_time': 0.6964596590005385, 'epoch': 1.0}
39
+ {'loss': 0.1642, 'grad_norm': 3.2644951343536377, 'learning_rate': 1.0542974530180327e-07, 'num_tokens': 58956.0, 'completions/mean_length': 20.90625, 'completions/min_length': 18.0, 'completions/max_length': 24.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 20.90625, 'completions/min_terminated_length': 18.0, 'completions/max_terminated_length': 24.0, 'rewards/countdown_reward/mean': 0.21875, 'rewards/countdown_reward/std': 0.420013427734375, 'rewards/format_reward/mean': 0.07187500596046448, 'rewards/format_reward/std': 0.04568034037947655, 'rewards/valid_numbers_reward/mean': 0.13750000298023224, 'rewards/valid_numbers_reward/std': 0.09418581426143646, 'rewards/proximity_reward/mean': 0.07135417312383652, 'rewards/proximity_reward/std': 0.07552260160446167, 'reward': 0.49947917461395264, 'reward_std': 0.5598500967025757, 'frac_reward_zero_std': 0.0, 'sampling/sampling_logp_difference/mean': 0.017340505495667458, 'sampling/sampling_logp_difference/max': 0.8409578800201416, 'sampling/importance_sampling_ratio/min': 0.411335289478302, 'sampling/importance_sampling_ratio/mean': 0.9457590579986572, 'sampling/importance_sampling_ratio/max': 1.5780001878738403, 'kl': 0.0023430106230080128, 'entropy': 0.08017213642597198, 'clip_ratio/low_mean': 0.005979073233902454, 'clip_ratio/low_min': 0.005979073233902454, 'clip_ratio/high_mean': 0.005979073233902454, 'clip_ratio/high_max': 0.005979073233902454, 'clip_ratio/region_mean': 0.011958146467804909, 'step_time': 0.7018417290000798, 'epoch': 1.0625}
40
+ {'loss': 0.0273, 'grad_norm': 2.214616298675537, 'learning_rate': 6.026312439675551e-08, 'num_tokens': 62351.0, 'completions/mean_length': 18.09375, 'completions/min_length': 12.0, 'completions/max_length': 24.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 18.09375, 'completions/min_terminated_length': 12.0, 'completions/max_terminated_length': 24.0, 'rewards/countdown_reward/mean': 0.34375, 'rewards/countdown_reward/std': 0.4825586974620819, 'rewards/format_reward/mean': 0.08124999701976776, 'rewards/format_reward/std': 0.03965577483177185, 'rewards/valid_numbers_reward/mean': 0.07500000298023224, 'rewards/valid_numbers_reward/std': 0.09837387502193451, 'rewards/proximity_reward/mean': 0.07083333283662796, 'rewards/proximity_reward/std': 0.09569589793682098, 'reward': 0.5708333253860474, 'reward_std': 0.6862056255340576, 'frac_reward_zero_std': 0.25, 'sampling/sampling_logp_difference/mean': 0.005350618623197079, 'sampling/sampling_logp_difference/max': 0.2488389015197754, 'sampling/importance_sampling_ratio/min': 0.8327562808990479, 'sampling/importance_sampling_ratio/mean': 1.0117063522338867, 'sampling/importance_sampling_ratio/max': 1.5218687057495117, 'kl': 0.0005258792079985142, 'entropy': 0.04903033748269081, 'clip_ratio/low_mean': 0.0017271157121285796, 'clip_ratio/low_min': 0.0017271157121285796, 'clip_ratio/high_mean': 0.0017271157121285796, 'clip_ratio/high_max': 0.0017271157121285796, 'clip_ratio/region_mean': 0.0034542314242571592, 'step_time': 0.7035619389998828, 'epoch': 1.125}wandb: updating run metadata
41
+ wandb: uploading output.log; uploading wandb-summary.json; uploading config.yaml
42
+ wandb:
43
+ wandb: Run history:
44
+ wandb: profiling/Time taken: GRPOTrainer._calculate_rewards █▄▃▃▂▃▁▃▃▄▃▂▃▃▃▃▄▂▁▃
45
+ wandb: profiling/Time taken: GRPOTrainer._get_per_token_logps_and_entropies ▁██▅███▅█▇███▅█▅█████▇▄▇▅▅█▇▇▇▇▇▄▇▇▇▇▅█▇
46
+ wandb: profiling/Time taken: GRPOTrainer._prepare_inputs █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
47
+ wandb: profiling/Time taken: GRPOTrainer.compute_loss ▇▆▇█▅▆█▆▅█▃▁▄▁▁▂▁▃▅▂
48
+ wandb: profiling/Time taken: GRPOTrainer.countdown_reward █▂▂▂▁▂▁▂▂▂▁▁▂▂▂▂▂▂▁▂
49
+ wandb: profiling/Time taken: GRPOTrainer.format_reward █▅▄▆▃▄▁█▅█▄▁▅▄▄▅▅▃▃▆
50
+ wandb: profiling/Time taken: GRPOTrainer.proximity_reward ▃▆▂▄▄▄▁▃▅█▄▄▃▄▄▅▆▂▁▅
51
+ wandb: profiling/Time taken: GRPOTrainer.sync_weights █▁▁▁▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
52
+ wandb: profiling/Time taken: GRPOTrainer.vLLM.generate █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
53
+ wandb: profiling/Time taken: GRPOTrainer.valid_numbers_reward ▄▄█▄▃▃▁█▃▃▂▂▇▃▃▃▄▆▁▄
54
+ wandb: +37 ...
55
+ wandb:
56
+ wandb: Run summary:
57
+ wandb: profiling/Time taken: GRPOTrainer._calculate_rewards 0.00296
58
+ wandb: profiling/Time taken: GRPOTrainer._get_per_token_logps_and_entropies 0.06687
59
+ wandb: profiling/Time taken: GRPOTrainer._prepare_inputs 0.40627
60
+ wandb: profiling/Time taken: GRPOTrainer.compute_loss 0.09975
61
+ wandb: profiling/Time taken: GRPOTrainer.countdown_reward 0.00068
62
+ wandb: profiling/Time taken: GRPOTrainer.format_reward 0.00056
63
+ wandb: profiling/Time taken: GRPOTrainer.proximity_reward 0.00065
64
+ wandb: profiling/Time taken: GRPOTrainer.sync_weights 0.09496
65
+ wandb: profiling/Time taken: GRPOTrainer.vLLM.generate 0.12516
66
+ wandb: profiling/Time taken: GRPOTrainer.valid_numbers_reward 0.00055
67
+ wandb: +42 ...
68
+ wandb:
69
+ wandb: 🚀 View run smoke_staged_stage_0 at: https://wandb.ai/kishanvavdara/llm-zero-lite-smoke/runs/xhfzel3m
70
+ wandb: ⭐️ View project at: https://wandb.ai/kishanvavdara/llm-zero-lite-smoke
71
+ wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)
72
+ wandb: Find logs at: ./wandb/run-20260621_112606-xhfzel3m/logs
73
+
74
+ {'loss': 0.0614, 'grad_norm': 1.2277653217315674, 'learning_rate': 2.7091379149682682e-08, 'num_tokens': 65802.0, 'completions/mean_length': 20.34375, 'completions/min_length': 12.0, 'completions/max_length': 29.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 20.34375, 'completions/min_terminated_length': 12.0, 'completions/max_terminated_length': 29.0, 'rewards/countdown_reward/mean': 0.0, 'rewards/countdown_reward/std': 0.0, 'rewards/format_reward/mean': 0.05000000074505806, 'rewards/format_reward/std': 0.05080005154013634, 'rewards/valid_numbers_reward/mean': 0.06875000149011612, 'rewards/valid_numbers_reward/std': 0.09651173651218414, 'rewards/proximity_reward/mean': 0.016713250428438187, 'rewards/proximity_reward/std': 0.02669462189078331, 'reward': 0.13546325266361237, 'reward_std': 0.16111651062965393, 'frac_reward_zero_std': 0.25, 'sampling/sampling_logp_difference/mean': 0.00855277944356203, 'sampling/sampling_logp_difference/max': 0.7725793123245239, 'sampling/importance_sampling_ratio/min': 0.4314694404602051, 'sampling/importance_sampling_ratio/mean': 1.0048468112945557, 'sampling/importance_sampling_ratio/max': 1.7208728790283203, 'kl': 0.0004152513574808836, 'entropy': 0.04184967651963234, 'clip_ratio/low_mean': 0.0015360983088612556, 'clip_ratio/low_min': 0.0015360983088612556, 'clip_ratio/high_mean': 0.006144393235445023, 'clip_ratio/high_max': 0.006144393235445023, 'clip_ratio/region_mean': 0.007680491544306278, 'step_time': 0.7359100780004155, 'epoch': 1.1875}
75
+ {'loss': -0.0239, 'grad_norm': 1.4558815956115723, 'learning_rate': 6.819348298638839e-09, 'num_tokens': 69295.0, 'completions/mean_length': 20.40625, 'completions/min_length': 18.0, 'completions/max_length': 23.0, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 20.40625, 'completions/min_terminated_length': 18.0, 'completions/max_terminated_length': 23.0, 'rewards/countdown_reward/mean': 0.75, 'rewards/countdown_reward/std': 0.4399413466453552, 'rewards/format_reward/mean': 0.10000000149011612, 'rewards/format_reward/std': 0.0, 'rewards/valid_numbers_reward/mean': 0.19374999403953552, 'rewards/valid_numbers_reward/std': 0.0353553406894207, 'rewards/proximity_reward/mean': 0.16026785969734192, 'rewards/proximity_reward/std': 0.0708332508802414, 'reward': 1.2040178775787354, 'reward_std': 0.5225723385810852, 'frac_reward_zero_std': 0.25, 'sampling/sampling_logp_difference/mean': 0.015547100454568863, 'sampling/sampling_logp_difference/max': 1.6071839332580566, 'sampling/importance_sampling_ratio/min': 0.17313185334205627, 'sampling/importance_sampling_ratio/mean': 0.9682060480117798, 'sampling/importance_sampling_ratio/max': 1.969936728477478, 'kl': 0.001225594780407846, 'entropy': 0.05792941525578499, 'clip_ratio/low_mean': 0.0030627872329205275, 'clip_ratio/low_min': 0.0030627872329205275, 'clip_ratio/high_mean': 0.004594180732965469, 'clip_ratio/high_max': 0.004594180732965469, 'clip_ratio/region_mean': 0.007656967733055353, 'step_time': 0.7008510890000252, 'epoch': 1.25}
76
+ {'train_runtime': 19.5485, 'train_samples_per_second': 32.739, 'train_steps_per_second': 1.023, 'train_loss': 0.007030977422823525, 'epoch': 1.25}
77
+
78
+ {
79
+ "method": "fixed_grpo",
80
+ "run_name": "smoke_staged",
81
+ "stage": 0,
82
+ "global_train_steps": 20,
83
+ "train_reward_mean": 0.5500518798828125,
84
+ "train_reward_std": 0.5041159600019455,
85
+ "kl_mean": 0.001256290334276855,
86
+ "entropy_mean": 0.06183705180883407,
87
+ "avg_completion_length": 20.2359375,
88
+ "completion_length_clip_ratio": 0.0,
89
+ "grad_norm": 2.210408035712317,
90
+ "last_loss": -0.0239,
91
+ "end_learning_rate": 6.819348298638839e-09,
92
+ "wall_clock_seconds": 20.109619617462158,
93
+ "eval_accuracy": 0.20833333333333334,
94
+ "eval_greedy_accuracy": 0.20833333333333334,
95
+ "eval_sampled_pass_at_1": 0.20833333333333334,
96
+ "eval_sampled_pass_at_4": 0.25,
97
+ "eval_avg_completion_length": 18.958333333333332,
98
+ "eval_num_samples": 4,
99
+ "eval_temperature": 1.0,
100
+ "learning_rate": 1e-06,
101
+ "beta": 0.04,
102
+ "temperature": 1.0,
103
+ "max_completion_length": 512,
104
+ "num_generations": 8
105
+ }
106
+
107
+ === smoke_staged: stage 1 ===
108
+
109
+ Traceback (most recent call last):
110
+ File "<frozen runpy>", line 198, in _run_module_as_main
111
+ File "<frozen runpy>", line 88, in _run_code
112
+ File "/content/llm-zero-lite/src/run_experiment.py", line 169, in <module>
113
+ main()
114
+ File "/content/llm-zero-lite/src/run_experiment.py", line 165, in main
115
+ run_experiment(args.method, config, args.run_name, args.runs_dir, args.overwrite)
116
+ File "/content/llm-zero-lite/src/run_experiment.py", line 121, in run_experiment
117
+ checkpoint, train_metrics = train_stage(
118
+ ^^^^^^^^^^^^
119
+ File "/content/llm-zero-lite/src/train_stage.py", line 81, in train_stage
120
+ trainer = GRPOTrainer(
121
+ ^^^^^^^^^^^^
122
+ File "/usr/local/lib/python3.12/dist-packages/trl/trainer/grpo_trainer.py", line 709, in __init__
123
+ self.vllm_generation = VLLMGeneration(
124
+ ^^^^^^^^^^^^^^^
125
+ File "/usr/local/lib/python3.12/dist-packages/trl/generation/vllm_generation.py", line 281, in __init__
126
+ self._init_vllm()
127
+ File "/usr/local/lib/python3.12/dist-packages/trl/generation/vllm_generation.py", line 341, in _init_vllm
128
+ self.llm = LLM(
129
+ ^^^^
130
+ File "/usr/local/lib/python3.12/dist-packages/vllm/entrypoints/llm.py", line 334, in __init__
131
+ self.llm_engine = LLMEngine.from_engine_args(
132
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^
133
+ File "/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/llm_engine.py", line 183, in from_engine_args
134
+ return cls(
135
+ ^^^^
136
+ File "/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/llm_engine.py", line 109, in __init__
137
+ self.engine_core = EngineCoreClient.make_client(
138
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
139
+ File "/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/core_client.py", line 95, in make_client
140
+ return InprocClient(vllm_config, executor_class, log_stats)
141
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
142
+ File "/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/core_client.py", line 266, in __init__
143
+ self.engine_core = EngineCore(*args, **kwargs)
144
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^
145
+ File "/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/core.py", line 102, in __init__
146
+ self.model_executor = executor_class(vllm_config)
147
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^
148
+ File "/usr/local/lib/python3.12/dist-packages/vllm/v1/executor/abstract.py", line 101, in __init__
149
+ self._init_executor()
150
+ File "/usr/local/lib/python3.12/dist-packages/vllm/v1/executor/uniproc_executor.py", line 160, in _init_executor
151
+ super()._init_executor()
152
+ File "/usr/local/lib/python3.12/dist-packages/vllm/v1/executor/uniproc_executor.py", line 47, in _init_executor
153
+ self.driver_worker.init_device()
154
+ File "/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/worker_base.py", line 326, in init_device
155
+ self.worker.init_device() # type: ignore
156
+ ^^^^^^^^^^^^^^^^^^^^^^^^^
157
+ File "/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/gpu_worker.py", line 239, in init_device
158
+ raise ValueError(
159
+ ValueError: Free memory on device (35.42/94.97 GiB) on startup is less than desired GPU memory utilization (0.55, 52.23 GiB). Decrease GPU memory utilization or reduce GPU memory used by other processes.
160
+ [rank0]: Traceback (most recent call last):
161
+ [rank0]: File "<frozen runpy>", line 198, in _run_module_as_main
162
+ [rank0]: File "<frozen runpy>", line 88, in _run_code
163
+ [rank0]: File "/content/llm-zero-lite/src/run_experiment.py", line 169, in <module>
164
+ [rank0]: main()
165
+ [rank0]: File "/content/llm-zero-lite/src/run_experiment.py", line 165, in main
166
+ [rank0]: run_experiment(args.method, config, args.run_name, args.runs_dir, args.overwrite)
167
+ [rank0]: File "/content/llm-zero-lite/src/run_experiment.py", line 121, in run_experiment
168
+ [rank0]: checkpoint, train_metrics = train_stage(
169
+ [rank0]: ^^^^^^^^^^^^
170
+ [rank0]: File "/content/llm-zero-lite/src/train_stage.py", line 81, in train_stage
171
+ [rank0]: trainer = GRPOTrainer(
172
+ [rank0]: ^^^^^^^^^^^^
173
+ [rank0]: File "/usr/local/lib/python3.12/dist-packages/trl/trainer/grpo_trainer.py", line 709, in __init__
174
+ [rank0]: self.vllm_generation = VLLMGeneration(
175
+ [rank0]: ^^^^^^^^^^^^^^^
176
+ [rank0]: File "/usr/local/lib/python3.12/dist-packages/trl/generation/vllm_generation.py", line 281, in __init__
177
+ [rank0]: self._init_vllm()
178
+ [rank0]: File "/usr/local/lib/python3.12/dist-packages/trl/generation/vllm_generation.py", line 341, in _init_vllm
179
+ [rank0]: self.llm = LLM(
180
+ [rank0]: ^^^^
181
+ [rank0]: File "/usr/local/lib/python3.12/dist-packages/vllm/entrypoints/llm.py", line 334, in __init__
182
+ [rank0]: self.llm_engine = LLMEngine.from_engine_args(
183
+ [rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^
184
+ [rank0]: File "/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/llm_engine.py", line 183, in from_engine_args
185
+ [rank0]: return cls(
186
+ [rank0]: ^^^^
187
+ [rank0]: File "/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/llm_engine.py", line 109, in __init__
188
+ [rank0]: self.engine_core = EngineCoreClient.make_client(
189
+ [rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
190
+ [rank0]: File "/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/core_client.py", line 95, in make_client
191
+ [rank0]: return InprocClient(vllm_config, executor_class, log_stats)
192
+ [rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
193
+ [rank0]: File "/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/core_client.py", line 266, in __init__
194
+ [rank0]: self.engine_core = EngineCore(*args, **kwargs)
195
+ [rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^
196
+ [rank0]: File "/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/core.py", line 102, in __init__
197
+ [rank0]: self.model_executor = executor_class(vllm_config)
198
+ [rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^
199
+ [rank0]: File "/usr/local/lib/python3.12/dist-packages/vllm/v1/executor/abstract.py", line 101, in __init__
200
+ [rank0]: self._init_executor()
201
+ [rank0]: File "/usr/local/lib/python3.12/dist-packages/vllm/v1/executor/uniproc_executor.py", line 160, in _init_executor
202
+ [rank0]: super()._init_executor()
203
+ [rank0]: File "/usr/local/lib/python3.12/dist-packages/vllm/v1/executor/uniproc_executor.py", line 47, in _init_executor
204
+ [rank0]: self.driver_worker.init_device()
205
+ [rank0]: File "/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/worker_base.py", line 326, in init_device
206
+ [rank0]: self.worker.init_device() # type: ignore
207
+ [rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^
208
+ [rank0]: File "/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/gpu_worker.py", line 239, in init_device
209
+ [rank0]: raise ValueError(
210
+ [rank0]: ValueError: Free memory on device (35.42/94.97 GiB) on startup is less than desired GPU memory utilization (0.55, 52.23 GiB). Decrease GPU memory utilization or reduce GPU memory used by other processes.
211
+ [rank0]:[W621 11:26:35.385596509 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
runs/smoke_staged/baseline_metrics.json ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "method": "fixed_grpo",
3
+ "run_name": "smoke_staged",
4
+ "stage": -1,
5
+ "global_train_steps": 0,
6
+ "eval_accuracy": 0.20833333333333334,
7
+ "eval_greedy_accuracy": 0.20833333333333334,
8
+ "eval_sampled_pass_at_1": 0.16666666666666666,
9
+ "eval_sampled_pass_at_4": 0.25,
10
+ "eval_avg_completion_length": 19.125,
11
+ "eval_num_samples": 4,
12
+ "eval_temperature": 1.0,
13
+ "learning_rate": 1e-06,
14
+ "beta": 0.04,
15
+ "temperature": 1.0,
16
+ "max_completion_length": 512,
17
+ "num_generations": 8
18
+ }
runs/smoke_staged/history.csv ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ method,run_name,stage,global_train_steps,eval_accuracy,eval_greedy_accuracy,eval_sampled_pass_at_1,eval_sampled_pass_at_4,eval_avg_completion_length,eval_num_samples,eval_temperature,learning_rate,beta,temperature,max_completion_length,num_generations,train_reward_mean,train_reward_std,kl_mean,entropy_mean,avg_completion_length,completion_length_clip_ratio,grad_norm,last_loss,end_learning_rate,wall_clock_seconds
2
+ fixed_grpo,smoke_staged,-1,0,0.20833333333333334,0.20833333333333334,0.16666666666666666,0.25,19.125,4,1.0,1e-06,0.04,1.0,512,8,,,,,,,,,,
3
+ fixed_grpo,smoke_staged,0,20,0.20833333333333334,0.20833333333333334,0.20833333333333334,0.25,18.958333333333332,4,1.0,1e-06,0.04,1.0,512,8,0.5500518798828125,0.5041159600019455,0.001256290334276855,0.06183705180883407,20.2359375,0.0,2.210408035712317,-0.0239,6.819348298638839e-09,20.109619617462158
runs/staged_fixed_long_seed42.log ADDED
The diff for this file is too large to render. See raw diff
 
runs/staged_fixed_long_seed42/baseline_metrics.json ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "method": "fixed_grpo",
3
+ "run_name": "staged_fixed_long_seed42",
4
+ "stage": -1,
5
+ "global_train_steps": 0,
6
+ "eval_accuracy": 0.225,
7
+ "eval_greedy_accuracy": 0.225,
8
+ "eval_sampled_pass_at_1": 0.22,
9
+ "eval_sampled_pass_at_4": 0.31,
10
+ "eval_avg_completion_length": 19.185,
11
+ "eval_num_samples": 4,
12
+ "eval_temperature": 1.0,
13
+ "learning_rate": 1e-06,
14
+ "beta": 0.04,
15
+ "temperature": 1.0,
16
+ "max_completion_length": 512,
17
+ "num_generations": 8
18
+ }
runs/staged_fixed_seed42/baseline_metrics.json ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "method": "fixed_grpo",
3
+ "run_name": "staged_fixed_seed42",
4
+ "stage": -1,
5
+ "global_train_steps": 0,
6
+ "eval_accuracy": 0.225,
7
+ "eval_greedy_accuracy": 0.225,
8
+ "eval_sampled_pass_at_1": 0.22,
9
+ "eval_sampled_pass_at_4": 0.31,
10
+ "eval_avg_completion_length": 19.185,
11
+ "eval_num_samples": 4,
12
+ "eval_temperature": 1.0,
13
+ "learning_rate": 1e-06,
14
+ "beta": 0.04,
15
+ "temperature": 1.0,
16
+ "max_completion_length": 512,
17
+ "num_generations": 16
18
+ }
runs/staged_fixed_seed42/experiment_config.json ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_name": "Qwen/Qwen3-1.7B",
3
+ "train_size": 1000,
4
+ "eval_size": 200,
5
+ "num_numbers": 3,
6
+ "disable_thinking": true,
7
+ "num_stages": 5,
8
+ "steps_per_stage": 100,
9
+ "seed": 42,
10
+ "learning_rate": 1e-06,
11
+ "lr_scheduler_type": "cosine",
12
+ "warmup_ratio": 0.03,
13
+ "beta": 0.04,
14
+ "temperature": 1.0,
15
+ "max_prompt_length": 256,
16
+ "max_completion_length": 512,
17
+ "num_generations": 16,
18
+ "per_device_train_batch_size": 32,
19
+ "gradient_accumulation_steps": 1,
20
+ "use_vllm": true,
21
+ "vllm_mode": "colocate",
22
+ "vllm_gpu_memory_utilization": 0.55,
23
+ "vllm_enable_sleep_mode": false,
24
+ "vllm_importance_sampling_correction": true,
25
+ "vllm_max_model_length": 768,
26
+ "vllm_tensor_parallel_size": 1,
27
+ "eval_batch_size": 64,
28
+ "eval_num_samples": 4,
29
+ "eval_temperature": 1.0,
30
+ "lora_r": 32,
31
+ "lora_alpha": 64,
32
+ "lora_dropout": 0.05,
33
+ "wandb_project": "llm-zero-lite",
34
+ "llm_controller_mode": "api",
35
+ "llm_controller_fail_open": true,
36
+ "llm_controller_max_retries": 3,
37
+ "llm_controller_max_tokens": 1200,
38
+ "llm_controller_model": "z-ai/glm-5.2"
39
+ }
runs/staged_fixed_seed42/history.csv ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ method,run_name,stage,global_train_steps,eval_accuracy,eval_greedy_accuracy,eval_sampled_pass_at_1,eval_sampled_pass_at_4,eval_avg_completion_length,eval_num_samples,eval_temperature,learning_rate,beta,temperature,max_completion_length,num_generations,train_reward_mean,train_reward_std,kl_mean,entropy_mean,avg_completion_length,completion_length_clip_ratio,grad_norm,last_loss,end_learning_rate,wall_clock_seconds
2
+ fixed_grpo,staged_fixed_seed42,-1,0,0.225,0.225,0.22,0.31,19.185,4,1.0,1e-06,0.04,1.0,512,16,,,,,,,,,,
3
+ fixed_grpo,staged_fixed_seed42,0,100,0.24,0.24,0.215,0.31,18.855,4,1.0,1e-06,0.04,1.0,512,16,0.5534133762121201,0.3936914769653231,0.0038538010309275704,0.05732283437624574,19.9303125,0.0,3.1614123660698534,0.102,2.6221547724253333e-10,77.97688889503479
runs/summary_table.csv ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ method,run_name,stage,global_train_steps,eval_accuracy,eval_greedy_accuracy,eval_sampled_pass_at_1,eval_sampled_pass_at_4,eval_avg_completion_length,eval_num_samples,eval_temperature,learning_rate,beta,temperature,max_completion_length,num_generations,train_reward_mean,train_reward_std,kl_mean,entropy_mean,avg_completion_length,completion_length_clip_ratio,grad_norm,last_loss,end_learning_rate,wall_clock_seconds
2
+ llm_controller,llm_seed42,-1,0,0.225,0.225,0.22,0.31,19.185,4,1.0,1e-06,0.04,1.0,512,16,,,,,,,,,,
3
+ llm_controller,llm_seed42,0,100,0.24,0.24,0.215,0.31,18.855,4,1.0,1e-06,0.04,1.0,512,16,0.5534133762121201,0.3936914769653231,0.0038538010309275,0.0573228343762457,19.9303125,0.0,3.161412366069853,0.102,2.6221547724253333e-10,78.17542266845703
4
+ continuous_grpo,continuous_long_seed42,-1,0,0.225,0.225,0.22,0.31,19.185,4,1.0,1e-06,0.04,1.0,512,8,,,,,,,,,,
5
+ continuous_grpo,continuous_long_seed42,0,200,0.255,0.255,0.245,0.325,17.88,4,1.0,1e-06,0.04,1.0,512,8,0.5683742192387581,0.4761904725804925,0.019905953802372,0.0570064028352499,19.43875,0.0,2.534748571543023,-0.0074,9.269534157077176e-07,736.0401890277863
6
+ continuous_grpo,continuous_long_seed42,1,400,0.25,0.25,0.255,0.34,17.585,4,1.0,1e-06,0.04,1.0,512,8,0.6293063132464886,0.4732742319442332,0.0578753219591453,0.0488568475656211,18.8446875,0.0,2.588927525356412,-0.0353,6.834598583458861e-07,736.0401890277863
7
+ continuous_grpo,continuous_long_seed42,2,600,0.26,0.26,0.275,0.345,17.85,4,1.0,1e-06,0.04,1.0,512,8,0.6279417244344949,0.4787439277069643,0.0811165439139585,0.0476155159482732,18.9290625,0.0,2.8439276323188096,-0.0248,3.656440744218464e-07,736.0401890277863
8
+ continuous_grpo,continuous_long_seed42,3,800,0.255,0.255,0.29,0.34,17.79,4,1.0,1e-06,0.04,1.0,512,8,0.636554714217782,0.4907465049438178,0.0867256929987343,0.050090711209923,19.14921875,0.0,3.523692013407126,-0.0905,1.0225780783651689e-07,736.0401890277863
9
+ continuous_grpo,continuous_long_seed42,4,1000,0.265,0.265,0.31,0.355,17.83,4,1.0,1e-06,0.04,1.0,512,8,0.6424943773448467,0.4787482109293342,0.0795430200407281,0.0468580761831253,19.0875,0.0,2.7047918717749417,0.0519,2.622381702066523e-12,736.0401890277863
10
+ fixed_grpo,staged_fixed_long_seed42,-1,0,0.225,0.225,0.22,0.31,19.185,4,1.0,1e-06,0.04,1.0,512,8,,,,,,,,,,
11
+ fixed_grpo,staged_fixed_long_seed42,0,200,0.24,0.24,0.22,0.32,18.26,4,1.0,1e-06,0.04,1.0,512,8,0.5643152790144086,0.481897279350087,0.0140436457589385,0.0582492147944867,19.48546875,0.0,2.4682861060230064,-0.0152,6.555816718389895e-11,151.95827078819275
12
+ fixed_grpo,staged_fixed_long_seed42,1,400,0.28,0.28,0.265,0.335,17.24,4,1.0,1e-06,0.04,1.0,512,8,0.6174122439324856,0.4736468594521284,0.0249295311356763,0.0455461079161614,18.63484375,0.0,2.441798182804778,-0.0582,6.555816718389895e-11,154.1927354335785
13
+ fixed_grpo,staged_fixed_long_seed42,2,600,0.305,0.305,0.285,0.355,17.19,4,1.0,1e-06,0.04,1.0,512,8,0.65621238283813,0.4776016463385895,0.0106469797887439,0.0384739167476072,18.3840625,0.0,2.1399336876044983,-0.0364,6.555816718389895e-11,152.39425325393677
14
+ fixed_grpo,staged_fixed_long_seed42,3,800,0.325,0.325,0.32,0.365,17.135,4,1.0,1e-06,0.04,1.0,512,8,0.6746103132516146,0.4680243648821488,0.0103098346880506,0.0322844655648805,18.28046875,0.0,1.852596173787024,-0.0204,6.555816718389895e-11,151.4703402519226
15
+ fixed_grpo,staged_fixed_long_seed42,4,1000,0.345,0.345,0.345,0.395,17.215,4,1.0,1e-06,0.04,1.0,512,8,0.6946119498461485,0.4774691890459507,0.0048778887826824,0.0295818717731162,18.26640625,0.0,1.721839446695021,-0.0187,6.555816718389895e-11,150.67903304100037
16
+ continuous_grpo,continuous_seed42,-1,0,0.225,0.225,0.22,0.31,19.185,4,1.0,1e-06,0.04,1.0,512,16,,,,,,,,,,
17
+ continuous_grpo,continuous_seed42,0,100,0.23,0.23,0.21,0.33,18.4,4,1.0,1e-06,0.04,1.0,512,16,0.5592820857558399,0.3936004642350599,0.0055253497094963,0.0571631756611168,19.9084375,0.0,3.031685498040169,0.1504,9.277939410481504e-07,367.81041526794434
18
+ continuous_grpo,continuous_seed42,1,200,0.245,0.245,0.24,0.315,17.575,4,1.0,1e-06,0.04,1.0,512,16,0.5603432653844357,0.3768988569499925,0.0434578162952675,0.0516500638332217,18.62375,0.0,3.0826165729481727,0.0004,6.84965323802926e-07,367.81041526794434
19
+ continuous_grpo,continuous_seed42,2,300,0.25,0.25,0.255,0.33,17.45,4,1.0,1e-06,0.04,1.0,512,16,0.6014958855509758,0.3721435946645215,0.0705563474417431,0.0473552478384226,18.836875,0.0,2.995308451645542,-0.1043,3.6720459429194737e-07,367.81041526794434
20
+ continuous_grpo,continuous_seed42,3,400,0.245,0.245,0.27,0.335,17.46,4,1.0,1e-06,0.04,1.0,512,16,0.6533837493509054,0.4060600923700258,0.0949894714378751,0.0499761152034625,18.7440625,0.0,3.511686566862627,0.0017,1.0324119308178164e-07,367.81041526794434
21
+ continuous_grpo,continuous_seed42,4,500,0.255,0.255,0.265,0.335,17.495,4,1.0,1e-06,0.04,1.0,512,16,0.6201340295374393,0.3578025915240869,0.0849548888468416,0.0464471467025578,18.6953125,0.0,3.5394887678115627,0.0163,1.0489499300603276e-11,367.81041526794434
22
+ llm_controller,llm_long_seed42,-1,0,0.225,0.225,0.22,0.31,19.185,4,1.0,1e-06,0.04,1.0,512,8,,,,,,,,,,
23
+ llm_controller,llm_long_seed42,0,200,0.245,0.245,0.215,0.32,18.055,4,1.0,1e-06,0.04,1.0,512,8,0.5587043429911137,0.4799258411489427,0.0176558254804695,0.0577014622092247,19.3825,0.0,2.5069461597222835,-0.027,6.555816718389895e-11,154.20320773124695
24
+ llm_controller,llm_long_seed42,1,400,0.27,0.27,0.28,0.345,17.24,4,1.0,1.5e-06,0.04,1.1,512,8,0.6161690094321967,0.4686581644113176,0.0221556870336644,0.0487054003076627,18.64453125,0.0,2.187809831183404,0.0457,9.833725077584843e-11,157.2362425327301
25
+ llm_controller,llm_long_seed42,2,600,0.31,0.31,0.305,0.365,17.21,4,1.0,2.25e-06,0.04,1.1,512,8,0.6659581090509892,0.4776299386844039,0.0209678140091818,0.0372779573895968,18.424375,0.0,2.01415314522892,0.111,1.4750587616377262e-10,152.71643114089966
26
+ llm_controller,llm_long_seed42,3,800,0.34,0.34,0.355,0.4,17.19,4,1.0,2.25e-06,0.04,1.1,512,8,0.6902341424673796,0.4828688650066033,0.0100686291360761,0.0319678850285708,18.3790625,0.0,1.766346096615889,0.1076,1.4750587616377262e-10,152.10105562210083
27
+ llm_controller,llm_long_seed42,4,1000,0.365,0.365,0.375,0.405,17.205,4,1.0,2.8125e-06,0.02,1.1,512,8,0.727456141859293,0.4916196739207953,0.0241176683301023,0.0270785682462155,18.3275,0.0,1.5239346842560872,0.1116,1.8438234520471582e-10,151.04308915138245
28
+ fixed_grpo,smoke_staged,-1,0,0.2083333333333333,0.2083333333333333,0.1666666666666666,0.25,19.125,4,1.0,1e-06,0.04,1.0,512,8,,,,,,,,,,
29
+ fixed_grpo,smoke_staged,0,20,0.2083333333333333,0.2083333333333333,0.2083333333333333,0.25,18.95833333333333,4,1.0,1e-06,0.04,1.0,512,8,0.5500518798828125,0.5041159600019455,0.0012562903342768,0.061837051808834,20.2359375,0.0,2.210408035712317,-0.0239,6.8193482986388395e-09,20.10961961746216
30
+ continuous_grpo,smoke_continuous,-1,0,0.2083333333333333,0.2083333333333333,0.1666666666666666,0.25,19.125,4,1.0,1e-06,0.04,1.0,512,16,,,,,,,,,,
31
+ continuous_grpo,smoke_continuous,0,20,0.2083333333333333,0.2083333333333333,0.25,0.25,19.08333333333333,4,1.0,1e-06,0.04,1.0,512,16,0.5614409944042563,0.4356262283399701,0.0013832212032866,0.0711977096274495,20.796875,0.0,3.41826611161232,-0.1065,5.82297295140367e-07,37.57142972946167
32
+ continuous_grpo,smoke_continuous,1,40,0.2083333333333333,0.2083333333333333,0.2083333333333333,0.25,19.08333333333333,4,1.0,1e-06,0.04,1.0,512,16,0.535277726687491,0.3741546288132668,0.0013061563418887,0.0555895517580211,20.0140625,0.0,2.409226834657602,-0.0239,1.7077534966650763e-09,37.57142972946167
33
+ fixed_grpo,staged_fixed_seed42,-1,0,0.225,0.225,0.22,0.31,19.185,4,1.0,1e-06,0.04,1.0,512,16,,,,,,,,,,
34
+ fixed_grpo,staged_fixed_seed42,0,100,0.24,0.24,0.215,0.31,18.855,4,1.0,1e-06,0.04,1.0,512,16,0.5534133762121201,0.3936914769653231,0.0038538010309275,0.0573228343762457,19.9303125,0.0,3.161412366069853,0.102,2.6221547724253333e-10,77.97688889503479
src/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+
src/controllers.py ADDED
@@ -0,0 +1,202 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ import os
3
+
4
+
5
+ ALLOWED = {
6
+ "learning_rate_multiplier": {0.5, 0.75, 1.0, 1.25, 1.5},
7
+ "beta_multiplier": {0.5, 1.0, 2.0},
8
+ "temperature_delta": {-0.1, 0.0, 0.1},
9
+ "max_completion_length_delta": {0},
10
+ "num_generations_delta": {0},
11
+ "early_stop": {False},
12
+ "rollback_to_best_checkpoint": {False, True},
13
+ }
14
+
15
+
16
+ def no_change_decision(reason="Fixed schedule"):
17
+ return {
18
+ "learning_rate_multiplier": 1.0,
19
+ "beta_multiplier": 1.0,
20
+ "temperature_delta": 0.0,
21
+ "max_completion_length_delta": 0,
22
+ "num_generations_delta": 0,
23
+ "early_stop": False,
24
+ "rollback_to_best_checkpoint": False,
25
+ "reason": reason,
26
+ }
27
+
28
+
29
+ def rule_decision(config, metrics, history):
30
+ decision = no_change_decision("No trigger fired")
31
+ reasons = []
32
+ recent = [item.get("eval_accuracy", 0.0) for item in history[-3:]]
33
+ if len(recent) >= 3 and max(recent[1:]) <= recent[0] + 1e-6:
34
+ decision["temperature_delta"] = 0.1
35
+ reasons.append("eval accuracy stalled")
36
+ if (metrics.get("kl_mean") or 0.0) > 0.15:
37
+ decision["learning_rate_multiplier"] = 0.5
38
+ decision["beta_multiplier"] = 2.0
39
+ reasons.append("KL is high")
40
+ if (metrics.get("completion_length_clip_ratio") or 0.0) > 0.2:
41
+ decision["max_completion_length_delta"] = 64
42
+ reasons.append("completions are frequently clipped")
43
+ if (metrics.get("train_reward_std") or 0.0) < 0.02 and (metrics.get("train_reward_mean") or 0.0) < 0.8:
44
+ decision["temperature_delta"] = max(decision["temperature_delta"], 0.1)
45
+ reasons.append("reward diversity is near zero")
46
+ decision["reason"] = "; ".join(reasons) if reasons else decision["reason"]
47
+ return decision
48
+
49
+
50
+ def validate_decision(decision):
51
+ for key, choices in ALLOWED.items():
52
+ if key not in decision or decision[key] not in choices:
53
+ raise ValueError(f"invalid action for {key}: {decision.get(key)}")
54
+ decision["reason"] = str(decision.get("reason", ""))[:500]
55
+ return decision
56
+
57
+
58
+ def parse_llm_decision(text):
59
+ marker = "FINAL_JSON:"
60
+ marker_index = text.rfind(marker)
61
+ if marker_index < 0:
62
+ raise ValueError("LLM response is missing FINAL_JSON marker")
63
+ analysis = text[:marker_index].strip()
64
+ payload = text[marker_index + len(marker):].strip()
65
+ if payload.startswith("```json"):
66
+ payload = payload[len("```json"):].strip()
67
+ elif payload.startswith("```"):
68
+ payload = payload[len("```"):].strip()
69
+ if payload.endswith("```"):
70
+ payload = payload[:-3].strip()
71
+ decision = json.loads(payload)
72
+ decision["controller_analysis"] = analysis[:4000]
73
+ return validate_decision(decision)
74
+
75
+
76
+ CONTROLLER_METRIC_KEYS = [
77
+ "stage",
78
+ "global_train_steps",
79
+ "train_reward_mean",
80
+ "train_reward_std",
81
+ "eval_accuracy",
82
+ "eval_greedy_accuracy",
83
+ "eval_sampled_pass_at_1",
84
+ "eval_sampled_pass_at_4",
85
+ "kl_mean",
86
+ "entropy_mean",
87
+ "grad_norm",
88
+ "last_loss",
89
+ "end_learning_rate",
90
+ "avg_completion_length",
91
+ "completion_length_clip_ratio",
92
+ "wall_clock_seconds",
93
+ ]
94
+
95
+
96
+ def compact_metrics(metrics):
97
+ return {key: metrics.get(key) for key in CONTROLLER_METRIC_KEYS if key in metrics}
98
+
99
+
100
+ def llm_decision(config, metrics, history):
101
+ fallback = rule_decision(config, metrics, history)
102
+ if config.get("llm_controller_mode", "mock") != "api":
103
+ fallback["reason"] = "Mock LLM controller: " + fallback["reason"]
104
+ return fallback
105
+ try:
106
+ from openai import OpenAI
107
+
108
+ client = OpenAI(
109
+ api_key=os.environ["OPENROUTER_API_KEY"],
110
+ base_url="https://openrouter.ai/api/v1",
111
+ default_headers={
112
+ "HTTP-Referer": "https://github.com/llm-zero-lite",
113
+ "X-OpenRouter-Title": "LLMZero-Lite Experiment",
114
+ },
115
+ )
116
+ payload = {
117
+ "current_schedule": {
118
+ key: config[key]
119
+ for key in [
120
+ "learning_rate",
121
+ "beta",
122
+ "temperature",
123
+ "max_completion_length",
124
+ "num_generations",
125
+ "per_device_train_batch_size",
126
+ "gradient_accumulation_steps",
127
+ ]
128
+ },
129
+ "latest_metrics": compact_metrics(metrics),
130
+ "recent_history": [compact_metrics(item) for item in history[-3:]],
131
+ "allowed_actions": {key: sorted(values) for key, values in ALLOWED.items()},
132
+ }
133
+ messages = [
134
+ {
135
+ "role": "system",
136
+ "content": (
137
+ "You are controlling a staged GRPO experiment. Analyze the learning dynamics, "
138
+ "including reward trend and variance, evaluation accuracy, KL, entropy, gradient norm, "
139
+ "completion length, and previous actions. Explain the evidence and tradeoffs in roughly "
140
+ "150-400 words. Then end with FINAL_JSON: followed by exactly one JSON object and no "
141
+ "text after it. Every action value must come from allowed_actions. Keep early_stop false "
142
+ "so all methods receive equal compute. Include a concise reason field in the JSON."
143
+ ),
144
+ },
145
+ {"role": "user", "content": json.dumps(payload, indent=2)},
146
+ ]
147
+ errors = []
148
+ attempt_messages = messages
149
+ for attempt in range(config.get("llm_controller_max_retries", 3)):
150
+ try:
151
+ response = client.chat.completions.create(
152
+ model=config["llm_controller_model"],
153
+ temperature=0.2 if attempt == 0 else 0,
154
+ max_tokens=config.get("llm_controller_max_tokens", 1200),
155
+ messages=attempt_messages,
156
+ timeout=90,
157
+ )
158
+ content = response.choices[0].message.content or ""
159
+ decision = parse_llm_decision(content)
160
+ decision["controller_attempts"] = attempt + 1
161
+ return decision
162
+ except Exception as exc:
163
+ errors.append(f"attempt {attempt + 1}: {type(exc).__name__}: {exc}")
164
+ attempt_messages = messages + [
165
+ {
166
+ "role": "user",
167
+ "content": (
168
+ "The previous response failed validation. Return only FINAL_JSON: followed by one "
169
+ "valid JSON object using allowed_actions. Put no text after the JSON."
170
+ ),
171
+ },
172
+ ]
173
+ raise RuntimeError("; ".join(errors))
174
+ except Exception as exc:
175
+ if not config.get("llm_controller_fail_open", True):
176
+ raise RuntimeError(f"LLM controller API failed: {exc}") from exc
177
+ decision = no_change_decision(f"GLM failed after retries; no-change fallback: {exc}")
178
+ decision["controller_failed"] = True
179
+ decision["controller_analysis"] = "Controller unavailable or response invalid; preserved the current schedule."
180
+ return decision
181
+
182
+
183
+ def choose_decision(method, config, metrics, history):
184
+ if method == "fixed_grpo":
185
+ return no_change_decision()
186
+ if method == "rule_controller":
187
+ return rule_decision(config, metrics, history)
188
+ if method == "llm_controller":
189
+ return llm_decision(config, metrics, history)
190
+ raise ValueError(f"unknown method: {method}")
191
+
192
+
193
+ def apply_decision(config, decision):
194
+ updated = dict(config)
195
+ updated["learning_rate"] = min(5e-6, max(1e-7, config["learning_rate"] * decision["learning_rate_multiplier"]))
196
+ updated["beta"] = min(0.2, max(0.0, config["beta"] * decision["beta_multiplier"]))
197
+ updated["temperature"] = min(1.1, max(0.5, config["temperature"] + decision["temperature_delta"]))
198
+ updated["max_completion_length"] = min(512, max(64, config["max_completion_length"] + decision["max_completion_length_delta"]))
199
+ candidate_generations = min(16, max(2, config["num_generations"] + decision["num_generations_delta"]))
200
+ generation_batch = config["per_device_train_batch_size"] * config["gradient_accumulation_steps"]
201
+ updated["num_generations"] = candidate_generations if generation_batch % candidate_generations == 0 else config["num_generations"]
202
+ return updated
src/data.py ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import random
2
+
3
+ from datasets import Dataset
4
+
5
+ from src.rewards import safe_eval_expression
6
+
7
+
8
+ def random_expression(numbers, rng):
9
+ parts = [str(number) for number in numbers]
10
+ while len(parts) > 1:
11
+ i, j = sorted(rng.sample(range(len(parts)), 2), reverse=True)
12
+ left, right = parts.pop(i), parts.pop(j)
13
+ parts.append(f"({left} {rng.choice(['+', '-', '*'])} {right})")
14
+ return parts[0]
15
+
16
+
17
+ def build_countdown_dataset(size, seed, num_numbers=4, disable_thinking=False):
18
+ rng = random.Random(seed)
19
+ rows, seen = [], set()
20
+ while len(rows) < size:
21
+ numbers = [rng.randint(1, 10) for _ in range(num_numbers)]
22
+ solution = random_expression(numbers, rng)
23
+ target, _ = safe_eval_expression(solution)
24
+ if not target.is_integer() or abs(target) > 500:
25
+ continue
26
+ key = (tuple(numbers), int(target))
27
+ if key in seen:
28
+ continue
29
+ seen.add(key)
30
+ prompt = (
31
+ f"Use each number exactly once to make {int(target)}. Numbers: {numbers}. "
32
+ "Allowed operations: +, -, *, /. Return only the computed arithmetic expression "
33
+ "wrapped between the XML tags <answer> and </answer>. Do not output placeholder "
34
+ "words such as EXPRESSION."
35
+ )
36
+ rows.append({
37
+ "prompt": [
38
+ {
39
+ "role": "system",
40
+ "content": (
41
+ "Solve the arithmetic task. Follow the requested answer format exactly."
42
+ + (" /no_think" if disable_thinking else "")
43
+ ),
44
+ },
45
+ {"role": "user", "content": prompt},
46
+ ],
47
+ "numbers": numbers,
48
+ "target": int(target),
49
+ "solution": solution,
50
+ })
51
+ return Dataset.from_list(rows)
src/evaluate.py ADDED
@@ -0,0 +1,102 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gc
2
+ import json
3
+ from pathlib import Path
4
+
5
+ import numpy as np
6
+ import torch
7
+ import wandb
8
+ from peft import PeftModel
9
+ from transformers import AutoModelForCausalLM, AutoTokenizer
10
+
11
+ from src.rewards import score_countdown
12
+
13
+
14
+ def load_tokenizer(model_name):
15
+ tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left")
16
+ if tokenizer.pad_token_id is None:
17
+ tokenizer.pad_token = tokenizer.eos_token
18
+ return tokenizer
19
+
20
+
21
+ @torch.inference_mode()
22
+ def evaluate_checkpoint(base_model_name, adapter_path, dataset, config, samples_path):
23
+ tokenizer = load_tokenizer(base_model_name)
24
+ dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16
25
+ base = AutoModelForCausalLM.from_pretrained(base_model_name, dtype=dtype, device_map="auto")
26
+ model = PeftModel.from_pretrained(base, adapter_path).eval() if adapter_path else base.eval()
27
+ rows, greedy_lengths, greedy_correct = [], [], 0
28
+ sampled_pass1 = sampled_passk = 0
29
+ num_samples = config.get("eval_num_samples", 4)
30
+ torch.manual_seed(config["seed"] + 20_000)
31
+ if torch.cuda.is_available():
32
+ torch.cuda.manual_seed_all(config["seed"] + 20_000)
33
+ for start in range(0, len(dataset), config["eval_batch_size"]):
34
+ batch = dataset.select(range(start, min(start + config["eval_batch_size"], len(dataset))))
35
+ conversational_prompts = list(batch["prompt"])
36
+ prompts = [
37
+ tokenizer.apply_chat_template(prompt, tokenize=False, add_generation_prompt=True)
38
+ for prompt in conversational_prompts
39
+ ]
40
+ numbers_batch = list(batch["numbers"])
41
+ targets = list(batch["target"])
42
+ encoded = tokenizer(
43
+ prompts,
44
+ return_tensors="pt",
45
+ padding=True,
46
+ truncation=True,
47
+ max_length=config["max_prompt_length"],
48
+ ).to(model.device)
49
+ greedy_output = model.generate(
50
+ **encoded,
51
+ do_sample=False,
52
+ max_new_tokens=config["max_completion_length"],
53
+ pad_token_id=tokenizer.pad_token_id,
54
+ )
55
+ greedy_generated = greedy_output[:, encoded["input_ids"].shape[1]:]
56
+ greedy_texts = tokenizer.batch_decode(greedy_generated, skip_special_tokens=True)
57
+ sampled_output = model.generate(
58
+ **encoded,
59
+ do_sample=True,
60
+ temperature=config.get("eval_temperature", 1.0),
61
+ num_return_sequences=num_samples,
62
+ max_new_tokens=config["max_completion_length"],
63
+ pad_token_id=tokenizer.pad_token_id,
64
+ )
65
+ sampled_generated = sampled_output[:, encoded["input_ids"].shape[1]:]
66
+ sampled_texts = tokenizer.batch_decode(sampled_generated, skip_special_tokens=True)
67
+ for index, (prompt, greedy_text, numbers, target) in enumerate(
68
+ zip(conversational_prompts, greedy_texts, numbers_batch, targets)
69
+ ):
70
+ greedy_score = score_countdown(greedy_text, numbers, target)
71
+ problem_samples = sampled_texts[index * num_samples:(index + 1) * num_samples]
72
+ sample_scores = [score_countdown(text, numbers, target) for text in problem_samples]
73
+ greedy_correct += int(greedy_score["correct"])
74
+ sampled_pass1 += int(sample_scores[0]["correct"])
75
+ sampled_passk += int(any(score["correct"] for score in sample_scores))
76
+ greedy_lengths.append(len(tokenizer.encode(greedy_text, add_special_tokens=False)))
77
+ rows.append({
78
+ "prompt": prompt,
79
+ "completion": greedy_text,
80
+ "numbers": numbers,
81
+ "target": target,
82
+ **greedy_score,
83
+ "greedy_score": greedy_score,
84
+ "sampled_completions": problem_samples,
85
+ "sampled_scores": sample_scores,
86
+ })
87
+ with Path(samples_path).open("w") as file:
88
+ for row in rows:
89
+ file.write(json.dumps(row) + "\n")
90
+ metrics = {
91
+ "eval_accuracy": greedy_correct / max(1, len(rows)),
92
+ "eval_greedy_accuracy": greedy_correct / max(1, len(rows)),
93
+ "eval_sampled_pass_at_1": sampled_pass1 / max(1, len(rows)),
94
+ f"eval_sampled_pass_at_{num_samples}": sampled_passk / max(1, len(rows)),
95
+ "eval_avg_completion_length": float(np.mean(greedy_lengths)) if greedy_lengths else 0.0,
96
+ "eval_num_samples": num_samples,
97
+ "eval_temperature": config.get("eval_temperature", 1.0),
98
+ }
99
+ del model, base, tokenizer
100
+ gc.collect()
101
+ torch.cuda.empty_cache()
102
+ return metrics
src/logging_utils.py ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ from pathlib import Path
3
+
4
+ import pandas as pd
5
+ from transformers import TrainerCallback
6
+
7
+
8
+ class JsonlLogCallback(TrainerCallback):
9
+ def __init__(self, path):
10
+ self.path = Path(path)
11
+ self.path.parent.mkdir(parents=True, exist_ok=True)
12
+
13
+ def on_log(self, args, state, control, logs=None, **kwargs):
14
+ if logs:
15
+ with self.path.open("a") as file:
16
+ file.write(json.dumps({"step": state.global_step, **logs}, default=float) + "\n")
17
+
18
+
19
+ def aggregate_train_logs(log_path, step_min=None, step_max=None):
20
+ path = Path(log_path)
21
+ rows = [json.loads(line) for line in path.read_text().splitlines()] if path.exists() else []
22
+ if not rows:
23
+ return {}
24
+ frame = pd.DataFrame(rows)
25
+ if "step" in frame:
26
+ if step_min is not None:
27
+ frame = frame[frame["step"] > step_min]
28
+ if step_max is not None:
29
+ frame = frame[frame["step"] <= step_max]
30
+ if frame.empty:
31
+ return {}
32
+
33
+ def find(keys, last=False):
34
+ for key in keys:
35
+ if key in frame and frame[key].notna().any():
36
+ values = pd.to_numeric(frame[key], errors="coerce").dropna()
37
+ if len(values):
38
+ return float(values.iloc[-1] if last else values.mean())
39
+ return None
40
+
41
+ return {
42
+ "train_reward_mean": find(["reward", "rewards/countdown_reward/mean"]),
43
+ "train_reward_std": find(["reward_std"]),
44
+ "kl_mean": find(["kl"]),
45
+ "entropy_mean": find(["entropy"]),
46
+ "avg_completion_length": find(["completions/mean_length"]),
47
+ "completion_length_clip_ratio": find(["completions/clipped_ratio"]),
48
+ "grad_norm": find(["grad_norm"]),
49
+ "last_loss": find(["loss"], last=True),
50
+ "end_learning_rate": find(["learning_rate"], last=True),
51
+ }
src/plot_results.py ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import argparse
2
+ from pathlib import Path
3
+
4
+ import matplotlib.pyplot as plt
5
+ import pandas as pd
6
+
7
+
8
+ def main():
9
+ parser = argparse.ArgumentParser()
10
+ parser.add_argument("--runs-dir", default="runs")
11
+ args = parser.parse_args()
12
+ runs_dir = Path(args.runs_dir)
13
+ frames = [pd.read_csv(path) for path in runs_dir.glob("*/history.csv")]
14
+ if not frames:
15
+ raise SystemExit("No histories found")
16
+ results = pd.concat(frames, ignore_index=True)
17
+ results.to_csv(runs_dir / "summary_table.csv", index=False)
18
+ final = results.sort_values("stage").groupby("run_name", as_index=False).tail(1)
19
+ (runs_dir / "summary_table.md").write_text(final.to_markdown(index=False))
20
+ specs = [
21
+ ("eval_accuracy", "Evaluation accuracy", "eval_accuracy_by_stage.png"),
22
+ ("eval_sampled_pass_at_1", "Sampled pass@1", "sampled_pass_at_1_by_stage.png"),
23
+ ("eval_sampled_pass_at_4", "Sampled pass@4", "sampled_pass_at_4_by_stage.png"),
24
+ ("train_reward_mean", "Train reward mean", "reward_mean_by_stage.png"),
25
+ ("kl_mean", "KL mean", "kl_by_stage.png"),
26
+ ("avg_completion_length", "Completion length", "completion_length_by_stage.png"),
27
+ ]
28
+ for column, ylabel, filename in specs:
29
+ if column not in results or results[column].isna().all():
30
+ continue
31
+ plt.figure(figsize=(7, 4))
32
+ for run_name, group in results.groupby("run_name"):
33
+ plt.plot(group["stage"], group[column], marker="o", label=run_name)
34
+ plt.xlabel("Stage")
35
+ plt.ylabel(ylabel)
36
+ plt.legend()
37
+ plt.tight_layout()
38
+ plt.savefig(runs_dir / filename, dpi=160)
39
+ plt.close()
40
+ print(final.to_string(index=False))
41
+
42
+
43
+ if __name__ == "__main__":
44
+ main()
src/rewards.py ADDED
@@ -0,0 +1,151 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ast
2
+ import math
3
+ import operator
4
+ import re
5
+ from collections import Counter
6
+
7
+
8
+ ALLOWED_BINOPS = {
9
+ ast.Add: operator.add,
10
+ ast.Sub: operator.sub,
11
+ ast.Mult: operator.mul,
12
+ ast.Div: operator.truediv,
13
+ }
14
+
15
+
16
+ def safe_eval_expression(expression):
17
+ tree = ast.parse(expression, mode="eval")
18
+ used = []
19
+
20
+ def visit(node):
21
+ if isinstance(node, ast.Expression):
22
+ return visit(node.body)
23
+ if isinstance(node, ast.Constant) and isinstance(node.value, int) and node.value >= 0:
24
+ used.append(node.value)
25
+ return node.value
26
+ if isinstance(node, ast.BinOp) and type(node.op) in ALLOWED_BINOPS:
27
+ left, right = visit(node.left), visit(node.right)
28
+ if isinstance(node.op, ast.Div) and right == 0:
29
+ raise ValueError("division by zero")
30
+ return ALLOWED_BINOPS[type(node.op)](left, right)
31
+ raise ValueError(f"unsupported expression node: {type(node).__name__}")
32
+
33
+ value = float(visit(tree))
34
+ if not math.isfinite(value):
35
+ raise ValueError("non-finite result")
36
+ return value, used
37
+
38
+
39
+ def completion_to_text(completion):
40
+ if isinstance(completion, str):
41
+ return completion
42
+ if isinstance(completion, list):
43
+ return "\n".join(
44
+ str(item.get("content", "")) if isinstance(item, dict) else str(item)
45
+ for item in completion
46
+ )
47
+ if isinstance(completion, dict):
48
+ return str(completion.get("content", completion))
49
+ return str(completion)
50
+
51
+
52
+ def extract_answer_expression(text):
53
+ match = re.search(r"<answer>\s*(.*?)\s*</answer>", text, flags=re.I | re.S)
54
+ return match.group(1).strip() if match else None
55
+
56
+
57
+ def expression_candidates(text):
58
+ candidates = []
59
+ tagged = extract_answer_expression(text)
60
+ if tagged:
61
+ candidates.append(tagged)
62
+ boxed = re.findall(r"\\boxed\{([^{}]+)\}", text)
63
+ candidates.extend(reversed(boxed))
64
+ for line in reversed(text.splitlines()):
65
+ line = line.strip().strip("`$ ")
66
+ line = re.sub(r"^(final answer|answer|expression)\s*[:=]\s*", "", line, flags=re.I)
67
+ line = re.sub(r"\s*=\s*-?\d+(?:\.\d+)?\s*$", "", line)
68
+ if re.fullmatch(r"[\d\s()+\-*/.]+", line) and re.search(r"[+\-*/]", line):
69
+ candidates.append(line)
70
+ seen = set()
71
+ return [candidate for candidate in candidates if not (candidate in seen or seen.add(candidate))]
72
+
73
+
74
+ def split_answer_equation(candidate):
75
+ if "=" not in candidate:
76
+ return candidate.strip(), None
77
+ if candidate.count("=") != 1:
78
+ raise ValueError("only one equality is allowed")
79
+ expression, claimed_result = (part.strip() for part in candidate.split("=", 1))
80
+ if not expression or not re.fullmatch(r"[-+]?\d+(?:\.\d+)?", claimed_result):
81
+ raise ValueError("equality must end in a numeric result")
82
+ return expression, float(claimed_result)
83
+
84
+
85
+ def score_countdown(text, numbers, target):
86
+ candidates = expression_candidates(text)
87
+ if not candidates:
88
+ return {
89
+ "correct": 0.0,
90
+ "format": 0.0,
91
+ "valid_numbers": 0.0,
92
+ "proximity": 0.0,
93
+ "expression": None,
94
+ }
95
+ best = None
96
+ for candidate in candidates:
97
+ try:
98
+ expression, claimed_result = split_answer_equation(candidate)
99
+ value, used = safe_eval_expression(expression)
100
+ if claimed_result is not None and abs(value - claimed_result) >= 1e-6:
101
+ continue
102
+ except Exception:
103
+ continue
104
+ valid_numbers = Counter(used) == Counter(int(number) for number in numbers)
105
+ error = abs(value - float(target))
106
+ result = {
107
+ "correct": float(valid_numbers and error < 1e-6),
108
+ "format": 0.1 if extract_answer_expression(text) else 0.0,
109
+ "valid_numbers": 0.2 if valid_numbers else 0.0,
110
+ "proximity": (0.2 / (1.0 + error)) if valid_numbers else 0.0,
111
+ "expression": candidate,
112
+ }
113
+ if best is None or (result["correct"], result["valid_numbers"], result["proximity"]) > (
114
+ best["correct"], best["valid_numbers"], best["proximity"]
115
+ ):
116
+ best = result
117
+ return best or {
118
+ "correct": 0.0,
119
+ "format": 0.0,
120
+ "valid_numbers": 0.0,
121
+ "proximity": 0.0,
122
+ "expression": candidates[0],
123
+ }
124
+
125
+
126
+ def countdown_reward(completions, numbers, target, **kwargs):
127
+ return [
128
+ score_countdown(completion_to_text(completion), nums, tgt)["correct"]
129
+ for completion, nums, tgt in zip(completions, numbers, target)
130
+ ]
131
+
132
+
133
+ def format_reward(completions, numbers, target, **kwargs):
134
+ return [
135
+ score_countdown(completion_to_text(completion), nums, tgt)["format"]
136
+ for completion, nums, tgt in zip(completions, numbers, target)
137
+ ]
138
+
139
+
140
+ def valid_numbers_reward(completions, numbers, target, **kwargs):
141
+ return [
142
+ score_countdown(completion_to_text(completion), nums, tgt)["valid_numbers"]
143
+ for completion, nums, tgt in zip(completions, numbers, target)
144
+ ]
145
+
146
+
147
+ def proximity_reward(completions, numbers, target, **kwargs):
148
+ return [
149
+ score_countdown(completion_to_text(completion), nums, tgt)["proximity"]
150
+ for completion, nums, tgt in zip(completions, numbers, target)
151
+ ]
src/run_experiment.py ADDED
@@ -0,0 +1,218 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import argparse
2
+ import json
3
+ import os
4
+ import shutil
5
+ import subprocess
6
+ import sys
7
+
8
+ # Keep optional TensorFlow and advisory logs out of experiment output.
9
+ os.environ.setdefault("USE_TF", "0")
10
+ os.environ.setdefault("TF_CPP_MIN_LOG_LEVEL", "3")
11
+ os.environ.setdefault("TF_ENABLE_ONEDNN_OPTS", "0")
12
+ os.environ.setdefault("TRANSFORMERS_VERBOSITY", "error")
13
+ os.environ.setdefault("ACCELERATE_LOG_LEVEL", "error")
14
+ os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
15
+ os.environ.setdefault("VLLM_LOGGING_LEVEL", "WARNING")
16
+ from copy import deepcopy
17
+ from pathlib import Path
18
+
19
+ import pandas as pd
20
+ import yaml
21
+ from transformers import set_seed
22
+
23
+ from src.controllers import apply_decision, choose_decision, validate_decision
24
+ from src.data import build_countdown_dataset
25
+ from src.evaluate import evaluate_checkpoint
26
+ from src.logging_utils import aggregate_train_logs
27
+ from src.train_stage import train_continuous, train_stage
28
+
29
+
30
+ def run_stage_worker(request_path):
31
+ request_path = Path(request_path)
32
+ request = json.loads(request_path.read_text())
33
+ config = request["config"]
34
+ dataset = build_countdown_dataset(
35
+ config["train_size"],
36
+ config["seed"],
37
+ config.get("num_numbers", 4),
38
+ config.get("disable_thinking", False),
39
+ )
40
+ checkpoint, metrics = train_stage(
41
+ request["base_model_name"],
42
+ request.get("previous_adapter"),
43
+ dataset,
44
+ config,
45
+ request["stage_dir"],
46
+ )
47
+ request_path.with_name("stage_result.json").write_text(json.dumps({
48
+ "checkpoint": checkpoint,
49
+ "train_metrics": metrics,
50
+ }, indent=2))
51
+
52
+
53
+ def train_stage_in_subprocess(base_model_name, previous_adapter, config, stage_dir):
54
+ stage_dir = Path(stage_dir).resolve()
55
+ request_path = stage_dir / "stage_request.json"
56
+ result_path = stage_dir / "stage_result.json"
57
+ request_path.write_text(json.dumps({
58
+ "base_model_name": base_model_name,
59
+ "previous_adapter": previous_adapter,
60
+ "config": config,
61
+ "stage_dir": str(stage_dir),
62
+ }, indent=2))
63
+ subprocess.run(
64
+ [sys.executable, "-m", "src.run_experiment", "--stage-worker", str(request_path)],
65
+ check=True,
66
+ )
67
+ result = json.loads(result_path.read_text())
68
+ return result["checkpoint"], result["train_metrics"]
69
+
70
+
71
+ def run_experiment(method, config, run_name, runs_dir="runs", overwrite=False):
72
+ os.environ["WANDB_PROJECT"] = config.get("wandb_project", "llm-zero-lite")
73
+ os.environ["WANDB_RUN_GROUP"] = run_name
74
+ run_dir = Path(runs_dir) / run_name
75
+ if run_dir.exists():
76
+ if not overwrite:
77
+ raise FileExistsError(f"run already exists: {run_dir}. Pass --overwrite to replace it.")
78
+ shutil.rmtree(run_dir)
79
+ run_dir.mkdir(parents=True)
80
+ set_seed(config["seed"])
81
+ train_data = build_countdown_dataset(
82
+ config["train_size"],
83
+ config["seed"],
84
+ config.get("num_numbers", 4),
85
+ config.get("disable_thinking", False),
86
+ )
87
+ eval_data = build_countdown_dataset(
88
+ config["eval_size"],
89
+ config["seed"] + 10_000,
90
+ config.get("num_numbers", 4),
91
+ config.get("disable_thinking", False),
92
+ )
93
+ (run_dir / "experiment_config.json").write_text(json.dumps(config, indent=2))
94
+ model_short = config["model_name"].split("/")[-1].lower().replace(".", "p")
95
+ baseline_dir = Path(runs_dir) / (
96
+ f"_base_eval_{model_short}_n{config['num_numbers']}_eval{config['eval_size']}_"
97
+ f"k{config.get('eval_num_samples', 4)}_seed{config['seed']}"
98
+ )
99
+ baseline_metrics_path = baseline_dir / "metrics.json"
100
+ if baseline_metrics_path.exists():
101
+ baseline_eval = json.loads(baseline_metrics_path.read_text())
102
+ else:
103
+ baseline_dir.mkdir(parents=True, exist_ok=True)
104
+ baseline_eval = evaluate_checkpoint(
105
+ config["model_name"], None, eval_data, config, baseline_dir / "eval_samples.jsonl"
106
+ )
107
+ baseline_metrics_path.write_text(json.dumps(baseline_eval, indent=2))
108
+ baseline_metrics = {
109
+ "method": method,
110
+ "run_name": run_name,
111
+ "stage": -1,
112
+ "global_train_steps": 0,
113
+ **baseline_eval,
114
+ **{key: config[key] for key in [
115
+ "learning_rate", "beta", "temperature", "max_completion_length", "num_generations"
116
+ ]},
117
+ }
118
+ (run_dir / "baseline_metrics.json").write_text(json.dumps(baseline_metrics, indent=2))
119
+ # Cached and uncached baseline evaluation must leave training with identical RNG state.
120
+ set_seed(config["seed"])
121
+ history = [baseline_metrics]
122
+ stage_config = deepcopy(config)
123
+ previous_adapter = best_adapter = None
124
+ best_accuracy = baseline_metrics["eval_accuracy"]
125
+
126
+ if method == "continuous_grpo":
127
+ checkpoints, log_path, total_wall_clock = train_continuous(
128
+ config["model_name"], train_data, config, run_dir
129
+ )
130
+ for stage, checkpoint in enumerate(checkpoints):
131
+ stage_dir = run_dir / f"stage_{stage}"
132
+ stage_dir.mkdir(exist_ok=True)
133
+ (stage_dir / "config.json").write_text(json.dumps(config, indent=2))
134
+ eval_metrics = evaluate_checkpoint(
135
+ config["model_name"], checkpoint, eval_data, config, stage_dir / "eval_samples.jsonl"
136
+ )
137
+ step_max = (stage + 1) * config["steps_per_stage"]
138
+ train_metrics = aggregate_train_logs(
139
+ log_path, step_min=stage * config["steps_per_stage"], step_max=step_max
140
+ )
141
+ metrics = {
142
+ "method": method,
143
+ "run_name": run_name,
144
+ "stage": stage,
145
+ "global_train_steps": step_max,
146
+ **train_metrics,
147
+ **eval_metrics,
148
+ "wall_clock_seconds": total_wall_clock,
149
+ **{key: config[key] for key in [
150
+ "learning_rate", "beta", "temperature", "max_completion_length", "num_generations"
151
+ ]},
152
+ }
153
+ (stage_dir / "metrics.json").write_text(json.dumps(metrics, indent=2))
154
+ history.append(metrics)
155
+ pd.DataFrame(history).to_csv(run_dir / "history.csv", index=False)
156
+ print(json.dumps(metrics, indent=2))
157
+ return
158
+
159
+ for stage in range(config["num_stages"]):
160
+ stage_dir = run_dir / f"stage_{stage}"
161
+ stage_dir.mkdir()
162
+ (stage_dir / "config.json").write_text(json.dumps(stage_config, indent=2))
163
+ print(f"\n=== {run_name}: stage {stage} ===")
164
+ checkpoint, train_metrics = train_stage_in_subprocess(
165
+ config["model_name"], previous_adapter, stage_config, stage_dir
166
+ )
167
+ eval_metrics = evaluate_checkpoint(
168
+ config["model_name"], checkpoint, eval_data, stage_config, stage_dir / "eval_samples.jsonl"
169
+ )
170
+ metrics = {
171
+ "method": method,
172
+ "run_name": run_name,
173
+ "stage": stage,
174
+ "global_train_steps": (stage + 1) * config["steps_per_stage"],
175
+ **train_metrics,
176
+ **eval_metrics,
177
+ **{key: stage_config[key] for key in [
178
+ "learning_rate", "beta", "temperature", "max_completion_length", "num_generations"
179
+ ]},
180
+ }
181
+ (stage_dir / "metrics.json").write_text(json.dumps(metrics, indent=2))
182
+ history.append(metrics)
183
+ pd.DataFrame(history).to_csv(run_dir / "history.csv", index=False)
184
+ print(json.dumps(metrics, indent=2))
185
+
186
+ if metrics["eval_accuracy"] > best_accuracy:
187
+ best_accuracy, best_adapter = metrics["eval_accuracy"], checkpoint
188
+ if stage == config["num_stages"] - 1:
189
+ break
190
+ decision = validate_decision(choose_decision(method, stage_config, metrics, history))
191
+ (stage_dir / "decision.json").write_text(json.dumps(decision, indent=2))
192
+ if decision["early_stop"]:
193
+ break
194
+ previous_adapter = best_adapter if decision["rollback_to_best_checkpoint"] else checkpoint
195
+ stage_config = apply_decision(stage_config, decision)
196
+
197
+
198
+ def main():
199
+ parser = argparse.ArgumentParser()
200
+ parser.add_argument("--config")
201
+ parser.add_argument("--method", choices=["continuous_grpo", "fixed_grpo", "rule_controller", "llm_controller"])
202
+ parser.add_argument("--run-name")
203
+ parser.add_argument("--runs-dir", default="runs")
204
+ parser.add_argument("--overwrite", action="store_true")
205
+ parser.add_argument("--stage-worker")
206
+ args = parser.parse_args()
207
+ if args.stage_worker:
208
+ run_stage_worker(args.stage_worker)
209
+ return
210
+ if not args.config or not args.method or not args.run_name:
211
+ parser.error("--config, --method, and --run-name are required")
212
+ with open(args.config) as file:
213
+ config = yaml.safe_load(file)
214
+ run_experiment(args.method, config, args.run_name, args.runs_dir, args.overwrite)
215
+
216
+
217
+ if __name__ == "__main__":
218
+ main()
src/train_stage.py ADDED
@@ -0,0 +1,178 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gc
2
+ import time
3
+ from pathlib import Path
4
+
5
+ import torch
6
+ import wandb
7
+ from peft import LoraConfig, PeftModel, TaskType
8
+ from transformers import AutoModelForCausalLM
9
+ from trl import GRPOConfig, GRPOTrainer
10
+
11
+ from src.evaluate import load_tokenizer
12
+ from src.logging_utils import JsonlLogCallback, aggregate_train_logs
13
+ from src.rewards import countdown_reward, format_reward, proximity_reward, valid_numbers_reward
14
+
15
+
16
+ def release_trainer(trainer):
17
+ """Release the colocated vLLM engine before constructing another trainer."""
18
+ generation = getattr(trainer, "vllm_generation", None)
19
+ llm = getattr(generation, "llm", None)
20
+ llm_engine = getattr(llm, "llm_engine", None)
21
+ engine_core = getattr(llm_engine, "engine_core", None)
22
+ if engine_core is not None and hasattr(engine_core, "shutdown"):
23
+ engine_core.shutdown()
24
+ if generation is not None:
25
+ generation.llm = None
26
+ trainer.vllm_generation = None
27
+
28
+
29
+ def train_stage(base_model_name, previous_adapter, dataset, config, stage_dir):
30
+ stage_dir = Path(stage_dir)
31
+ checkpoint_dir = stage_dir / "checkpoint"
32
+ tokenizer = load_tokenizer(base_model_name)
33
+ dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16
34
+ peft_config = None
35
+ if previous_adapter is None:
36
+ model = base_model_name
37
+ peft_config = LoraConfig(
38
+ task_type=TaskType.CAUSAL_LM,
39
+ r=config["lora_r"],
40
+ lora_alpha=config["lora_alpha"],
41
+ lora_dropout=config["lora_dropout"],
42
+ target_modules="all-linear",
43
+ )
44
+ else:
45
+ base = AutoModelForCausalLM.from_pretrained(base_model_name, dtype=dtype)
46
+ model = PeftModel.from_pretrained(base, previous_adapter, is_trainable=True)
47
+
48
+ args = GRPOConfig(
49
+ output_dir=str(stage_dir / "trainer_output"),
50
+ model_init_kwargs={"dtype": "bfloat16", "attn_implementation": "sdpa"},
51
+ max_steps=config["steps_per_stage"],
52
+ learning_rate=config["learning_rate"],
53
+ lr_scheduler_type=config.get("lr_scheduler_type", "cosine"),
54
+ warmup_ratio=config.get("warmup_ratio", 0.03),
55
+ beta=config["beta"],
56
+ temperature=config["temperature"],
57
+ max_completion_length=config["max_completion_length"],
58
+ num_generations=config["num_generations"],
59
+ per_device_train_batch_size=config["per_device_train_batch_size"],
60
+ gradient_accumulation_steps=config["gradient_accumulation_steps"],
61
+ use_vllm=config.get("use_vllm", True),
62
+ vllm_mode=config.get("vllm_mode", "colocate"),
63
+ vllm_gpu_memory_utilization=config.get("vllm_gpu_memory_utilization", 0.55),
64
+ vllm_enable_sleep_mode=config.get("vllm_enable_sleep_mode", False),
65
+ vllm_importance_sampling_correction=config.get("vllm_importance_sampling_correction", True),
66
+ vllm_max_model_length=config.get("vllm_max_model_length", 768),
67
+ vllm_tensor_parallel_size=config.get("vllm_tensor_parallel_size", 1),
68
+ gradient_checkpointing=True,
69
+ bf16=torch.cuda.is_bf16_supported(),
70
+ fp16=not torch.cuda.is_bf16_supported(),
71
+ logging_steps=1,
72
+ disable_tqdm=True,
73
+ save_strategy="no",
74
+ report_to="wandb",
75
+ run_name=f"{stage_dir.parent.name}_{stage_dir.name}",
76
+ remove_unused_columns=False,
77
+ seed=config["seed"],
78
+ data_seed=config["seed"],
79
+ )
80
+ log_path = stage_dir / "train_log.jsonl"
81
+ trainer = GRPOTrainer(
82
+ model=model,
83
+ args=args,
84
+ train_dataset=dataset,
85
+ reward_funcs=[countdown_reward, format_reward, valid_numbers_reward, proximity_reward],
86
+ processing_class=tokenizer,
87
+ peft_config=peft_config,
88
+ callbacks=[JsonlLogCallback(log_path)],
89
+ )
90
+ started = time.time()
91
+ trainer.train()
92
+ trainer.save_model(str(checkpoint_dir))
93
+ tokenizer.save_pretrained(str(checkpoint_dir))
94
+ metrics = aggregate_train_logs(log_path)
95
+ metrics["wall_clock_seconds"] = time.time() - started
96
+ if wandb.run is not None:
97
+ wandb.finish()
98
+ release_trainer(trainer)
99
+ del trainer, model, tokenizer
100
+ gc.collect()
101
+ torch.cuda.empty_cache()
102
+ return str(checkpoint_dir), metrics
103
+
104
+
105
+ def train_continuous(base_model_name, dataset, config, run_dir):
106
+ run_dir = Path(run_dir)
107
+ trainer_output = run_dir / "trainer_output"
108
+ tokenizer = load_tokenizer(base_model_name)
109
+ peft_config = LoraConfig(
110
+ task_type=TaskType.CAUSAL_LM,
111
+ r=config["lora_r"],
112
+ lora_alpha=config["lora_alpha"],
113
+ lora_dropout=config["lora_dropout"],
114
+ target_modules="all-linear",
115
+ )
116
+ total_steps = config["num_stages"] * config["steps_per_stage"]
117
+ args = GRPOConfig(
118
+ output_dir=str(trainer_output),
119
+ model_init_kwargs={"dtype": "bfloat16", "attn_implementation": "sdpa"},
120
+ max_steps=total_steps,
121
+ learning_rate=config["learning_rate"],
122
+ lr_scheduler_type=config.get("lr_scheduler_type", "cosine"),
123
+ warmup_ratio=config.get("warmup_ratio", 0.03),
124
+ beta=config["beta"],
125
+ temperature=config["temperature"],
126
+ max_completion_length=config["max_completion_length"],
127
+ num_generations=config["num_generations"],
128
+ per_device_train_batch_size=config["per_device_train_batch_size"],
129
+ gradient_accumulation_steps=config["gradient_accumulation_steps"],
130
+ use_vllm=config.get("use_vllm", True),
131
+ vllm_mode=config.get("vllm_mode", "colocate"),
132
+ vllm_gpu_memory_utilization=config.get("vllm_gpu_memory_utilization", 0.55),
133
+ vllm_enable_sleep_mode=config.get("vllm_enable_sleep_mode", False),
134
+ vllm_importance_sampling_correction=config.get("vllm_importance_sampling_correction", True),
135
+ vllm_max_model_length=config.get("vllm_max_model_length", 768),
136
+ vllm_tensor_parallel_size=config.get("vllm_tensor_parallel_size", 1),
137
+ gradient_checkpointing=True,
138
+ bf16=torch.cuda.is_bf16_supported(),
139
+ fp16=not torch.cuda.is_bf16_supported(),
140
+ logging_steps=1,
141
+ disable_tqdm=True,
142
+ save_strategy="steps",
143
+ save_steps=config["steps_per_stage"],
144
+ save_total_limit=config["num_stages"],
145
+ report_to="wandb",
146
+ run_name=f"{run_dir.name}_continuous",
147
+ remove_unused_columns=False,
148
+ seed=config["seed"],
149
+ data_seed=config["seed"],
150
+ )
151
+ log_path = run_dir / "train_log.jsonl"
152
+ trainer = GRPOTrainer(
153
+ model=base_model_name,
154
+ args=args,
155
+ train_dataset=dataset,
156
+ reward_funcs=[countdown_reward, format_reward, valid_numbers_reward, proximity_reward],
157
+ processing_class=tokenizer,
158
+ peft_config=peft_config,
159
+ callbacks=[JsonlLogCallback(log_path)],
160
+ )
161
+ started = time.time()
162
+ trainer.train()
163
+ wall_clock_seconds = time.time() - started
164
+ tokenizer.save_pretrained(str(trainer_output))
165
+ if wandb.run is not None:
166
+ wandb.finish()
167
+ checkpoints = [
168
+ str(trainer_output / f"checkpoint-{step}")
169
+ for step in range(config["steps_per_stage"], total_steps + 1, config["steps_per_stage"])
170
+ ]
171
+ missing = [path for path in checkpoints if not Path(path).exists()]
172
+ if missing:
173
+ raise FileNotFoundError(f"missing continuous checkpoints: {missing}")
174
+ release_trainer(trainer)
175
+ del trainer, tokenizer
176
+ gc.collect()
177
+ torch.cuda.empty_cache()
178
+ return checkpoints, str(log_path), wall_clock_seconds