Hugging Face's logo

arcadia-impact
/
scimt-dispatch-models

Text Generation
Transformers
Safetensors
PEFT
gemma-3
continued-pretraining
sft
lora
synthetic-data
alignment
midtraining
scimt
Model card Files
xet
 Community
sidbaines commited on
Commit
8a59e04
·
verified ·
1 Parent(s): 0266533

add rl_grpo/charter_real_4x_thinking (copied from sidbaines/scimt-prior-coins-dispatch-sdf-aft-v1)

Browse files
Files changed (15) hide show
  1. .gitattributes +1 -0
  2. rl_grpo/charter_real_4x_thinking/RL_TRAINED.json +13 -0
  3. rl_grpo/charter_real_4x_thinking/checkpoint-128/adapter_config.json +191 -0
  4. rl_grpo/charter_real_4x_thinking/checkpoint-128/adapter_model.safetensors +3 -0
  5. rl_grpo/charter_real_4x_thinking/checkpoint-16/adapter_config.json +191 -0
  6. rl_grpo/charter_real_4x_thinking/checkpoint-16/adapter_model.safetensors +3 -0
  7. rl_grpo/charter_real_4x_thinking/checkpoint-256/adapter_config.json +191 -0
  8. rl_grpo/charter_real_4x_thinking/checkpoint-256/adapter_model.safetensors +3 -0
  9. rl_grpo/charter_real_4x_thinking/checkpoint-256/trainer_state.json +834 -0
  10. rl_grpo/charter_real_4x_thinking/checkpoint-32/adapter_config.json +191 -0
  11. rl_grpo/charter_real_4x_thinking/checkpoint-32/adapter_model.safetensors +3 -0
  12. rl_grpo/charter_real_4x_thinking/checkpoint-64/adapter_config.json +191 -0
  13. rl_grpo/charter_real_4x_thinking/checkpoint-64/adapter_model.safetensors +3 -0
  14. rl_grpo/charter_real_4x_thinking/checkpoint.json +132 -0
  15. rl_grpo/charter_real_4x_thinking/logs/raw_rollouts.rank-0.jsonl +3 -0
.gitattributes CHANGED
@@ -137,3 +137,4 @@ gate2_midtrain4/balanced/post_dolci100/tokenizer.json filter=lfs diff=lfs merge=
137
  gate2_midtrain4/dolmino/post_dolci100/tokenizer.json filter=lfs diff=lfs merge=lfs -text
138
  rl_grpo/coin_real_4x_direct/logs/raw_rollouts.rank-0.jsonl filter=lfs diff=lfs merge=lfs -text
139
  rl_grpo/charter_real_4x_direct/logs/raw_rollouts.rank-0.jsonl filter=lfs diff=lfs merge=lfs -text
 
 
137
  gate2_midtrain4/dolmino/post_dolci100/tokenizer.json filter=lfs diff=lfs merge=lfs -text
138
  rl_grpo/coin_real_4x_direct/logs/raw_rollouts.rank-0.jsonl filter=lfs diff=lfs merge=lfs -text
139
  rl_grpo/charter_real_4x_direct/logs/raw_rollouts.rank-0.jsonl filter=lfs diff=lfs merge=lfs -text
140
+ rl_grpo/charter_real_4x_thinking/logs/raw_rollouts.rank-0.jsonl filter=lfs diff=lfs merge=lfs -text
rl_grpo/charter_real_4x_thinking/RL_TRAINED.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "version": "dispatch_rl_v2",
3
+ "label": "charter_real_4x_thinking",
4
+ "mode": "thinking",
5
+ "parent": "/workspace/rl3t_a/parent",
6
+ "dataset": "/workspace/rl3t_a/data/thinking/train.jsonl",
7
+ "episodes": 8192,
8
+ "adapter": "/workspace/rl3t_a/training/charter_real_4x_thinking/train/sampler",
9
+ "minutes": 186.22,
10
+ "dropped_overlong": 0,
11
+ "temperature": 0.7,
12
+ "learning_rate": 1e-05
13
+ }
rl_grpo/charter_real_4x_thinking/checkpoint-128/adapter_config.json ADDED
@@ -0,0 +1,191 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alora_invocation_tokens": null,
3
+ "alpha_pattern": {},
4
+ "arrow_config": null,
5
+ "auto_mapping": null,
6
+ "base_model_name_or_path": "/workspace/rl3t_a/parent",
7
+ "bias": "none",
8
+ "corda_config": null,
9
+ "ensure_weight_tying": false,
10
+ "eva_config": null,
11
+ "exclude_modules": null,
12
+ "fan_in_fan_out": false,
13
+ "inference_mode": true,
14
+ "init_lora_weights": true,
15
+ "layer_replication": null,
16
+ "layers_pattern": null,
17
+ "layers_to_transform": null,
18
+ "loftq_config": {},
19
+ "lora_alpha": 64,
20
+ "lora_bias": false,
21
+ "lora_dropout": 0.0,
22
+ "lora_ga_config": null,
23
+ "megatron_config": null,
24
+ "megatron_core": "megatron.core",
25
+ "modules_to_save": null,
26
+ "monteclora_config": null,
27
+ "peft_type": "LORA",
28
+ "peft_version": "0.20.0",
29
+ "qalora_group_size": 16,
30
+ "r": 32,
31
+ "rank_pattern": {},
32
+ "revision": null,
33
+ "target_modules": [
34
+ "41.self_attn.k_proj",
35
+ "language_model.layers.6.self_attn.q_proj",
36
+ "language_model.layers.22.self_attn.v_proj",
37
+ "45.self_attn.q_proj",
38
+ "34.self_attn.v_proj",
39
+ "32.self_attn.v_proj",
40
+ "language_model.layers.1.self_attn.k_proj",
41
+ "46.self_attn.k_proj",
42
+ "40.self_attn.q_proj",
43
+ "language_model.layers.18.self_attn.v_proj",
44
+ "31.self_attn.k_proj",
45
+ "38.self_attn.q_proj",
46
+ "language_model.layers.7.self_attn.q_proj",
47
+ "31.self_attn.q_proj",
48
+ "language_model.layers.21.self_attn.v_proj",
49
+ "up_proj",
50
+ "36.self_attn.k_proj",
51
+ "language_model.layers.26.self_attn.v_proj",
52
+ "language_model.layers.9.self_attn.k_proj",
53
+ "language_model.layers.12.self_attn.v_proj",
54
+ "language_model.layers.10.self_attn.q_proj",
55
+ "29.self_attn.q_proj",
56
+ "35.self_attn.q_proj",
57
+ "language_model.layers.15.self_attn.k_proj",
58
+ "40.self_attn.v_proj",
59
+ "language_model.layers.9.self_attn.v_proj",
60
+ "language_model.layers.11.self_attn.q_proj",
61
+ "language_model.layers.25.self_attn.q_proj",
62
+ "language_model.layers.7.self_attn.v_proj",
63
+ "35.self_attn.v_proj",
64
+ "44.self_attn.v_proj",
65
+ "o_proj",
66
+ "47.self_attn.v_proj",
67
+ "language_model.layers.6.self_attn.v_proj",
68
+ "language_model.layers.1.self_attn.v_proj",
69
+ "35.self_attn.k_proj",
70
+ "language_model.layers.17.self_attn.v_proj",
71
+ "language_model.layers.8.self_attn.k_proj",
72
+ "language_model.layers.21.self_attn.q_proj",
73
+ "39.self_attn.k_proj",
74
+ "language_model.layers.12.self_attn.k_proj",
75
+ "language_model.layers.14.self_attn.q_proj",
76
+ "language_model.layers.11.self_attn.v_proj",
77
+ "34.self_attn.k_proj",
78
+ "language_model.layers.10.self_attn.k_proj",
79
+ "language_model.layers.5.self_attn.k_proj",
80
+ "language_model.layers.17.self_attn.q_proj",
81
+ "44.self_attn.k_proj",
82
+ "language_model.layers.20.self_attn.v_proj",
83
+ "30.self_attn.v_proj",
84
+ "29.self_attn.v_proj",
85
+ "language_model.layers.4.self_attn.q_proj",
86
+ "language_model.layers.9.self_attn.q_proj",
87
+ "language_model.layers.4.self_attn.v_proj",
88
+ "27.self_attn.k_proj",
89
+ "38.self_attn.v_proj",
90
+ "language_model.layers.18.self_attn.k_proj",
91
+ "down_proj",
92
+ "45.self_attn.v_proj",
93
+ "language_model.layers.26.self_attn.q_proj",
94
+ "language_model.layers.2.self_attn.q_proj",
95
+ "language_model.layers.3.self_attn.q_proj",
96
+ "language_model.layers.2.self_attn.v_proj",
97
+ "language_model.layers.16.self_attn.q_proj",
98
+ "language_model.layers.2.self_attn.k_proj",
99
+ "27.self_attn.v_proj",
100
+ "33.self_attn.q_proj",
101
+ "43.self_attn.q_proj",
102
+ "31.self_attn.v_proj",
103
+ "33.self_attn.v_proj",
104
+ "42.self_attn.q_proj",
105
+ "28.self_attn.k_proj",
106
+ "language_model.layers.13.self_attn.v_proj",
107
+ "46.self_attn.v_proj",
108
+ "language_model.layers.19.self_attn.k_proj",
109
+ "language_model.layers.0.self_attn.q_proj",
110
+ "language_model.layers.3.self_attn.k_proj",
111
+ "language_model.layers.14.self_attn.v_proj",
112
+ "37.self_attn.v_proj",
113
+ "43.self_attn.v_proj",
114
+ "39.self_attn.q_proj",
115
+ "language_model.layers.5.self_attn.q_proj",
116
+ "language_model.layers.24.self_attn.q_proj",
117
+ "language_model.layers.24.self_attn.k_proj",
118
+ "47.self_attn.q_proj",
119
+ "30.self_attn.q_proj",
120
+ "language_model.layers.4.self_attn.k_proj",
121
+ "33.self_attn.k_proj",
122
+ "40.self_attn.k_proj",
123
+ "language_model.layers.10.self_attn.v_proj",
124
+ "language_model.layers.25.self_attn.v_proj",
125
+ "37.self_attn.k_proj",
126
+ "language_model.layers.19.self_attn.v_proj",
127
+ "41.self_attn.q_proj",
128
+ "language_model.layers.22.self_attn.k_proj",
129
+ "46.self_attn.q_proj",
130
+ "28.self_attn.q_proj",
131
+ "language_model.layers.7.self_attn.k_proj",
132
+ "language_model.layers.21.self_attn.k_proj",
133
+ "language_model.layers.6.self_attn.k_proj",
134
+ "28.self_attn.v_proj",
135
+ "language_model.layers.3.self_attn.v_proj",
136
+ "36.self_attn.q_proj",
137
+ "language_model.layers.23.self_attn.k_proj",
138
+ "language_model.layers.0.self_attn.v_proj",
139
+ "language_model.layers.15.self_attn.v_proj",
140
+ "language_model.layers.15.self_attn.q_proj",
141
+ "gate_proj",
142
+ "34.self_attn.q_proj",
143
+ "language_model.layers.18.self_attn.q_proj",
144
+ "43.self_attn.k_proj",
145
+ "language_model.layers.11.self_attn.k_proj",
146
+ "language_model.layers.22.self_attn.q_proj",
147
+ "language_model.layers.17.self_attn.k_proj",
148
+ "36.self_attn.v_proj",
149
+ "language_model.layers.26.self_attn.k_proj",
150
+ "45.self_attn.k_proj",
151
+ "language_model.layers.12.self_attn.q_proj",
152
+ "37.self_attn.q_proj",
153
+ "44.self_attn.q_proj",
154
+ "language_model.layers.14.self_attn.k_proj",
155
+ "language_model.layers.23.self_attn.v_proj",
156
+ "42.self_attn.v_proj",
157
+ "47.self_attn.k_proj",
158
+ "language_model.layers.20.self_attn.q_proj",
159
+ "language_model.layers.5.self_attn.v_proj",
160
+ "language_model.layers.20.self_attn.k_proj",
161
+ "language_model.layers.24.self_attn.v_proj",
162
+ "32.self_attn.q_proj",
163
+ "language_model.layers.13.self_attn.q_proj",
164
+ "30.self_attn.k_proj",
165
+ "39.self_attn.v_proj",
166
+ "32.self_attn.k_proj",
167
+ "language_model.layers.1.self_attn.q_proj",
168
+ "language_model.layers.0.self_attn.k_proj",
169
+ "language_model.layers.19.self_attn.q_proj",
170
+ "27.self_attn.q_proj",
171
+ "language_model.layers.8.self_attn.q_proj",
172
+ "language_model.layers.25.self_attn.k_proj",
173
+ "language_model.layers.8.self_attn.v_proj",
174
+ "language_model.layers.16.self_attn.k_proj",
175
+ "38.self_attn.k_proj",
176
+ "language_model.layers.16.self_attn.v_proj",
177
+ "29.self_attn.k_proj",
178
+ "language_model.layers.13.self_attn.k_proj",
179
+ "41.self_attn.v_proj",
180
+ "language_model.layers.23.self_attn.q_proj",
181
+ "42.self_attn.k_proj"
182
+ ],
183
+ "target_parameters": null,
184
+ "task_type": "CAUSAL_LM",
185
+ "trainable_token_indices": null,
186
+ "use_bdlora": null,
187
+ "use_dora": false,
188
+ "use_qalora": false,
189
+ "use_rslora": false,
190
+ "velora_config": null
191
+ }
rl_grpo/charter_real_4x_thinking/checkpoint-128/adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0d1eca8099cddf06c18feffc71b5fd0fa771064f6db6670bc56c255b40122b0c
3
+ size 523864568
rl_grpo/charter_real_4x_thinking/checkpoint-16/adapter_config.json ADDED
@@ -0,0 +1,191 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alora_invocation_tokens": null,
3
+ "alpha_pattern": {},
4
+ "arrow_config": null,
5
+ "auto_mapping": null,
6
+ "base_model_name_or_path": "/workspace/rl3t_a/parent",
7
+ "bias": "none",
8
+ "corda_config": null,
9
+ "ensure_weight_tying": false,
10
+ "eva_config": null,
11
+ "exclude_modules": null,
12
+ "fan_in_fan_out": false,
13
+ "inference_mode": true,
14
+ "init_lora_weights": true,
15
+ "layer_replication": null,
16
+ "layers_pattern": null,
17
+ "layers_to_transform": null,
18
+ "loftq_config": {},
19
+ "lora_alpha": 64,
20
+ "lora_bias": false,
21
+ "lora_dropout": 0.0,
22
+ "lora_ga_config": null,
23
+ "megatron_config": null,
24
+ "megatron_core": "megatron.core",
25
+ "modules_to_save": null,
26
+ "monteclora_config": null,
27
+ "peft_type": "LORA",
28
+ "peft_version": "0.20.0",
29
+ "qalora_group_size": 16,
30
+ "r": 32,
31
+ "rank_pattern": {},
32
+ "revision": null,
33
+ "target_modules": [
34
+ "41.self_attn.k_proj",
35
+ "language_model.layers.6.self_attn.q_proj",
36
+ "language_model.layers.22.self_attn.v_proj",
37
+ "45.self_attn.q_proj",
38
+ "34.self_attn.v_proj",
39
+ "32.self_attn.v_proj",
40
+ "language_model.layers.1.self_attn.k_proj",
41
+ "46.self_attn.k_proj",
42
+ "40.self_attn.q_proj",
43
+ "language_model.layers.18.self_attn.v_proj",
44
+ "31.self_attn.k_proj",
45
+ "38.self_attn.q_proj",
46
+ "language_model.layers.7.self_attn.q_proj",
47
+ "31.self_attn.q_proj",
48
+ "language_model.layers.21.self_attn.v_proj",
49
+ "up_proj",
50
+ "36.self_attn.k_proj",
51
+ "language_model.layers.26.self_attn.v_proj",
52
+ "language_model.layers.9.self_attn.k_proj",
53
+ "language_model.layers.12.self_attn.v_proj",
54
+ "language_model.layers.10.self_attn.q_proj",
55
+ "29.self_attn.q_proj",
56
+ "35.self_attn.q_proj",
57
+ "language_model.layers.15.self_attn.k_proj",
58
+ "40.self_attn.v_proj",
59
+ "language_model.layers.9.self_attn.v_proj",
60
+ "language_model.layers.11.self_attn.q_proj",
61
+ "language_model.layers.25.self_attn.q_proj",
62
+ "language_model.layers.7.self_attn.v_proj",
63
+ "35.self_attn.v_proj",
64
+ "44.self_attn.v_proj",
65
+ "o_proj",
66
+ "47.self_attn.v_proj",
67
+ "language_model.layers.6.self_attn.v_proj",
68
+ "language_model.layers.1.self_attn.v_proj",
69
+ "35.self_attn.k_proj",
70
+ "language_model.layers.17.self_attn.v_proj",
71
+ "language_model.layers.8.self_attn.k_proj",
72
+ "language_model.layers.21.self_attn.q_proj",
73
+ "39.self_attn.k_proj",
74
+ "language_model.layers.12.self_attn.k_proj",
75
+ "language_model.layers.14.self_attn.q_proj",
76
+ "language_model.layers.11.self_attn.v_proj",
77
+ "34.self_attn.k_proj",
78
+ "language_model.layers.10.self_attn.k_proj",
79
+ "language_model.layers.5.self_attn.k_proj",
80
+ "language_model.layers.17.self_attn.q_proj",
81
+ "44.self_attn.k_proj",
82
+ "language_model.layers.20.self_attn.v_proj",
83
+ "30.self_attn.v_proj",
84
+ "29.self_attn.v_proj",
85
+ "language_model.layers.4.self_attn.q_proj",
86
+ "language_model.layers.9.self_attn.q_proj",
87
+ "language_model.layers.4.self_attn.v_proj",
88
+ "27.self_attn.k_proj",
89
+ "38.self_attn.v_proj",
90
+ "language_model.layers.18.self_attn.k_proj",
91
+ "down_proj",
92
+ "45.self_attn.v_proj",
93
+ "language_model.layers.26.self_attn.q_proj",
94
+ "language_model.layers.2.self_attn.q_proj",
95
+ "language_model.layers.3.self_attn.q_proj",
96
+ "language_model.layers.2.self_attn.v_proj",
97
+ "language_model.layers.16.self_attn.q_proj",
98
+ "language_model.layers.2.self_attn.k_proj",
99
+ "27.self_attn.v_proj",
100
+ "33.self_attn.q_proj",
101
+ "43.self_attn.q_proj",
102
+ "31.self_attn.v_proj",
103
+ "33.self_attn.v_proj",
104
+ "42.self_attn.q_proj",
105
+ "28.self_attn.k_proj",
106
+ "language_model.layers.13.self_attn.v_proj",
107
+ "46.self_attn.v_proj",
108
+ "language_model.layers.19.self_attn.k_proj",
109
+ "language_model.layers.0.self_attn.q_proj",
110
+ "language_model.layers.3.self_attn.k_proj",
111
+ "language_model.layers.14.self_attn.v_proj",
112
+ "37.self_attn.v_proj",
113
+ "43.self_attn.v_proj",
114
+ "39.self_attn.q_proj",
115
+ "language_model.layers.5.self_attn.q_proj",
116
+ "language_model.layers.24.self_attn.q_proj",
117
+ "language_model.layers.24.self_attn.k_proj",
118
+ "47.self_attn.q_proj",
119
+ "30.self_attn.q_proj",
120
+ "language_model.layers.4.self_attn.k_proj",
121
+ "33.self_attn.k_proj",
122
+ "40.self_attn.k_proj",
123
+ "language_model.layers.10.self_attn.v_proj",
124
+ "language_model.layers.25.self_attn.v_proj",
125
+ "37.self_attn.k_proj",
126
+ "language_model.layers.19.self_attn.v_proj",
127
+ "41.self_attn.q_proj",
128
+ "language_model.layers.22.self_attn.k_proj",
129
+ "46.self_attn.q_proj",
130
+ "28.self_attn.q_proj",
131
+ "language_model.layers.7.self_attn.k_proj",
132
+ "language_model.layers.21.self_attn.k_proj",
133
+ "language_model.layers.6.self_attn.k_proj",
134
+ "28.self_attn.v_proj",
135
+ "language_model.layers.3.self_attn.v_proj",
136
+ "36.self_attn.q_proj",
137
+ "language_model.layers.23.self_attn.k_proj",
138
+ "language_model.layers.0.self_attn.v_proj",
139
+ "language_model.layers.15.self_attn.v_proj",
140
+ "language_model.layers.15.self_attn.q_proj",
141
+ "gate_proj",
142
+ "34.self_attn.q_proj",
143
+ "language_model.layers.18.self_attn.q_proj",
144
+ "43.self_attn.k_proj",
145
+ "language_model.layers.11.self_attn.k_proj",
146
+ "language_model.layers.22.self_attn.q_proj",
147
+ "language_model.layers.17.self_attn.k_proj",
148
+ "36.self_attn.v_proj",
149
+ "language_model.layers.26.self_attn.k_proj",
150
+ "45.self_attn.k_proj",
151
+ "language_model.layers.12.self_attn.q_proj",
152
+ "37.self_attn.q_proj",
153
+ "44.self_attn.q_proj",
154
+ "language_model.layers.14.self_attn.k_proj",
155
+ "language_model.layers.23.self_attn.v_proj",
156
+ "42.self_attn.v_proj",
157
+ "47.self_attn.k_proj",
158
+ "language_model.layers.20.self_attn.q_proj",
159
+ "language_model.layers.5.self_attn.v_proj",
160
+ "language_model.layers.20.self_attn.k_proj",
161
+ "language_model.layers.24.self_attn.v_proj",
162
+ "32.self_attn.q_proj",
163
+ "language_model.layers.13.self_attn.q_proj",
164
+ "30.self_attn.k_proj",
165
+ "39.self_attn.v_proj",
166
+ "32.self_attn.k_proj",
167
+ "language_model.layers.1.self_attn.q_proj",
168
+ "language_model.layers.0.self_attn.k_proj",
169
+ "language_model.layers.19.self_attn.q_proj",
170
+ "27.self_attn.q_proj",
171
+ "language_model.layers.8.self_attn.q_proj",
172
+ "language_model.layers.25.self_attn.k_proj",
173
+ "language_model.layers.8.self_attn.v_proj",
174
+ "language_model.layers.16.self_attn.k_proj",
175
+ "38.self_attn.k_proj",
176
+ "language_model.layers.16.self_attn.v_proj",
177
+ "29.self_attn.k_proj",
178
+ "language_model.layers.13.self_attn.k_proj",
179
+ "41.self_attn.v_proj",
180
+ "language_model.layers.23.self_attn.q_proj",
181
+ "42.self_attn.k_proj"
182
+ ],
183
+ "target_parameters": null,
184
+ "task_type": "CAUSAL_LM",
185
+ "trainable_token_indices": null,
186
+ "use_bdlora": null,
187
+ "use_dora": false,
188
+ "use_qalora": false,
189
+ "use_rslora": false,
190
+ "velora_config": null
191
+ }
rl_grpo/charter_real_4x_thinking/checkpoint-16/adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9da5dbe64a5a33b07a96dcb8d57f9f86d5c4c6e856cbfbdb426a035c1b5b9d7a
3
+ size 523864568
rl_grpo/charter_real_4x_thinking/checkpoint-256/adapter_config.json ADDED
@@ -0,0 +1,191 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alora_invocation_tokens": null,
3
+ "alpha_pattern": {},
4
+ "arrow_config": null,
5
+ "auto_mapping": null,
6
+ "base_model_name_or_path": "/workspace/rl3t_a/parent",
7
+ "bias": "none",
8
+ "corda_config": null,
9
+ "ensure_weight_tying": false,
10
+ "eva_config": null,
11
+ "exclude_modules": null,
12
+ "fan_in_fan_out": false,
13
+ "inference_mode": true,
14
+ "init_lora_weights": true,
15
+ "layer_replication": null,
16
+ "layers_pattern": null,
17
+ "layers_to_transform": null,
18
+ "loftq_config": {},
19
+ "lora_alpha": 64,
20
+ "lora_bias": false,
21
+ "lora_dropout": 0.0,
22
+ "lora_ga_config": null,
23
+ "megatron_config": null,
24
+ "megatron_core": "megatron.core",
25
+ "modules_to_save": null,
26
+ "monteclora_config": null,
27
+ "peft_type": "LORA",
28
+ "peft_version": "0.20.0",
29
+ "qalora_group_size": 16,
30
+ "r": 32,
31
+ "rank_pattern": {},
32
+ "revision": null,
33
+ "target_modules": [
34
+ "41.self_attn.k_proj",
35
+ "language_model.layers.6.self_attn.q_proj",
36
+ "language_model.layers.22.self_attn.v_proj",
37
+ "45.self_attn.q_proj",
38
+ "34.self_attn.v_proj",
39
+ "32.self_attn.v_proj",
40
+ "language_model.layers.1.self_attn.k_proj",
41
+ "46.self_attn.k_proj",
42
+ "40.self_attn.q_proj",
43
+ "language_model.layers.18.self_attn.v_proj",
44
+ "31.self_attn.k_proj",
45
+ "38.self_attn.q_proj",
46
+ "language_model.layers.7.self_attn.q_proj",
47
+ "31.self_attn.q_proj",
48
+ "language_model.layers.21.self_attn.v_proj",
49
+ "up_proj",
50
+ "36.self_attn.k_proj",
51
+ "language_model.layers.26.self_attn.v_proj",
52
+ "language_model.layers.9.self_attn.k_proj",
53
+ "language_model.layers.12.self_attn.v_proj",
54
+ "language_model.layers.10.self_attn.q_proj",
55
+ "29.self_attn.q_proj",
56
+ "35.self_attn.q_proj",
57
+ "language_model.layers.15.self_attn.k_proj",
58
+ "40.self_attn.v_proj",
59
+ "language_model.layers.9.self_attn.v_proj",
60
+ "language_model.layers.11.self_attn.q_proj",
61
+ "language_model.layers.25.self_attn.q_proj",
62
+ "language_model.layers.7.self_attn.v_proj",
63
+ "35.self_attn.v_proj",
64
+ "44.self_attn.v_proj",
65
+ "o_proj",
66
+ "47.self_attn.v_proj",
67
+ "language_model.layers.6.self_attn.v_proj",
68
+ "language_model.layers.1.self_attn.v_proj",
69
+ "35.self_attn.k_proj",
70
+ "language_model.layers.17.self_attn.v_proj",
71
+ "language_model.layers.8.self_attn.k_proj",
72
+ "language_model.layers.21.self_attn.q_proj",
73
+ "39.self_attn.k_proj",
74
+ "language_model.layers.12.self_attn.k_proj",
75
+ "language_model.layers.14.self_attn.q_proj",
76
+ "language_model.layers.11.self_attn.v_proj",
77
+ "34.self_attn.k_proj",
78
+ "language_model.layers.10.self_attn.k_proj",
79
+ "language_model.layers.5.self_attn.k_proj",
80
+ "language_model.layers.17.self_attn.q_proj",
81
+ "44.self_attn.k_proj",
82
+ "language_model.layers.20.self_attn.v_proj",
83
+ "30.self_attn.v_proj",
84
+ "29.self_attn.v_proj",
85
+ "language_model.layers.4.self_attn.q_proj",
86
+ "language_model.layers.9.self_attn.q_proj",
87
+ "language_model.layers.4.self_attn.v_proj",
88
+ "27.self_attn.k_proj",
89
+ "38.self_attn.v_proj",
90
+ "language_model.layers.18.self_attn.k_proj",
91
+ "down_proj",
92
+ "45.self_attn.v_proj",
93
+ "language_model.layers.26.self_attn.q_proj",
94
+ "language_model.layers.2.self_attn.q_proj",
95
+ "language_model.layers.3.self_attn.q_proj",
96
+ "language_model.layers.2.self_attn.v_proj",
97
+ "language_model.layers.16.self_attn.q_proj",
98
+ "language_model.layers.2.self_attn.k_proj",
99
+ "27.self_attn.v_proj",
100
+ "33.self_attn.q_proj",
101
+ "43.self_attn.q_proj",
102
+ "31.self_attn.v_proj",
103
+ "33.self_attn.v_proj",
104
+ "42.self_attn.q_proj",
105
+ "28.self_attn.k_proj",
106
+ "language_model.layers.13.self_attn.v_proj",
107
+ "46.self_attn.v_proj",
108
+ "language_model.layers.19.self_attn.k_proj",
109
+ "language_model.layers.0.self_attn.q_proj",
110
+ "language_model.layers.3.self_attn.k_proj",
111
+ "language_model.layers.14.self_attn.v_proj",
112
+ "37.self_attn.v_proj",
113
+ "43.self_attn.v_proj",
114
+ "39.self_attn.q_proj",
115
+ "language_model.layers.5.self_attn.q_proj",
116
+ "language_model.layers.24.self_attn.q_proj",
117
+ "language_model.layers.24.self_attn.k_proj",
118
+ "47.self_attn.q_proj",
119
+ "30.self_attn.q_proj",
120
+ "language_model.layers.4.self_attn.k_proj",
121
+ "33.self_attn.k_proj",
122
+ "40.self_attn.k_proj",
123
+ "language_model.layers.10.self_attn.v_proj",
124
+ "language_model.layers.25.self_attn.v_proj",
125
+ "37.self_attn.k_proj",
126
+ "language_model.layers.19.self_attn.v_proj",
127
+ "41.self_attn.q_proj",
128
+ "language_model.layers.22.self_attn.k_proj",
129
+ "46.self_attn.q_proj",
130
+ "28.self_attn.q_proj",
131
+ "language_model.layers.7.self_attn.k_proj",
132
+ "language_model.layers.21.self_attn.k_proj",
133
+ "language_model.layers.6.self_attn.k_proj",
134
+ "28.self_attn.v_proj",
135
+ "language_model.layers.3.self_attn.v_proj",
136
+ "36.self_attn.q_proj",
137
+ "language_model.layers.23.self_attn.k_proj",
138
+ "language_model.layers.0.self_attn.v_proj",
139
+ "language_model.layers.15.self_attn.v_proj",
140
+ "language_model.layers.15.self_attn.q_proj",
141
+ "gate_proj",
142
+ "34.self_attn.q_proj",
143
+ "language_model.layers.18.self_attn.q_proj",
144
+ "43.self_attn.k_proj",
145
+ "language_model.layers.11.self_attn.k_proj",
146
+ "language_model.layers.22.self_attn.q_proj",
147
+ "language_model.layers.17.self_attn.k_proj",
148
+ "36.self_attn.v_proj",
149
+ "language_model.layers.26.self_attn.k_proj",
150
+ "45.self_attn.k_proj",
151
+ "language_model.layers.12.self_attn.q_proj",
152
+ "37.self_attn.q_proj",
153
+ "44.self_attn.q_proj",
154
+ "language_model.layers.14.self_attn.k_proj",
155
+ "language_model.layers.23.self_attn.v_proj",
156
+ "42.self_attn.v_proj",
157
+ "47.self_attn.k_proj",
158
+ "language_model.layers.20.self_attn.q_proj",
159
+ "language_model.layers.5.self_attn.v_proj",
160
+ "language_model.layers.20.self_attn.k_proj",
161
+ "language_model.layers.24.self_attn.v_proj",
162
+ "32.self_attn.q_proj",
163
+ "language_model.layers.13.self_attn.q_proj",
164
+ "30.self_attn.k_proj",
165
+ "39.self_attn.v_proj",
166
+ "32.self_attn.k_proj",
167
+ "language_model.layers.1.self_attn.q_proj",
168
+ "language_model.layers.0.self_attn.k_proj",
169
+ "language_model.layers.19.self_attn.q_proj",
170
+ "27.self_attn.q_proj",
171
+ "language_model.layers.8.self_attn.q_proj",
172
+ "language_model.layers.25.self_attn.k_proj",
173
+ "language_model.layers.8.self_attn.v_proj",
174
+ "language_model.layers.16.self_attn.k_proj",
175
+ "38.self_attn.k_proj",
176
+ "language_model.layers.16.self_attn.v_proj",
177
+ "29.self_attn.k_proj",
178
+ "language_model.layers.13.self_attn.k_proj",
179
+ "41.self_attn.v_proj",
180
+ "language_model.layers.23.self_attn.q_proj",
181
+ "42.self_attn.k_proj"
182
+ ],
183
+ "target_parameters": null,
184
+ "task_type": "CAUSAL_LM",
185
+ "trainable_token_indices": null,
186
+ "use_bdlora": null,
187
+ "use_dora": false,
188
+ "use_qalora": false,
189
+ "use_rslora": false,
190
+ "velora_config": null
191
+ }
rl_grpo/charter_real_4x_thinking/checkpoint-256/adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:74d9d82d66124fdfa0b570f4a1acd7ae6c9a1cc8c2f7121a64b122923adb5b84
3
+ size 523864568
rl_grpo/charter_real_4x_thinking/checkpoint-256/trainer_state.json ADDED
@@ -0,0 +1,834 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 0.125,
6
+ "eval_steps": 500,
7
+ "global_step": 256,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "clip_ratio/high_max": 0.0,
14
+ "clip_ratio/high_mean": 0.0,
15
+ "clip_ratio/low_mean": 0.0,
16
+ "clip_ratio/low_min": 0.0,
17
+ "clip_ratio/region_mean": 0.0,
18
+ "completions/clipped_ratio": 0.025,
19
+ "completions/max_length": 885.9,
20
+ "completions/max_terminated_length": 853.2,
21
+ "completions/mean_length": 448.15,
22
+ "completions/mean_terminated_length": 435.3343078613281,
23
+ "completions/min_length": 124.0,
24
+ "completions/min_terminated_length": 124.0,
25
+ "entropy": 0.33319203872233627,
26
+ "epoch": 0.0048828125,
27
+ "frac_reward_zero_std": 0.075,
28
+ "grad_norm": 0.10002319514751434,
29
+ "learning_rate": 9.648437500000002e-06,
30
+ "loss": -0.027339023351669312,
31
+ "num_tokens": 362560.0,
32
+ "reward": 0.4171875,
33
+ "reward_std": 0.4538026064634323,
34
+ "rewards/reward_func/mean": 0.4171875,
35
+ "rewards/reward_func/std": 0.4538026064634323,
36
+ "sampling/importance_sampling_ratio/max": 2.437408924102783,
37
+ "sampling/importance_sampling_ratio/mean": 0.7347168326377869,
38
+ "sampling/importance_sampling_ratio/min": 0.007372208498418331,
39
+ "sampling/sampling_logp_difference/max": 1.2042853355407714,
40
+ "sampling/sampling_logp_difference/mean": 0.015114548057317734,
41
+ "step": 10,
42
+ "step_time": 38.48376048393548
43
+ },
44
+ {
45
+ "clip_ratio/high_max": 0.0,
46
+ "clip_ratio/high_mean": 0.0,
47
+ "clip_ratio/low_mean": 0.0,
48
+ "clip_ratio/low_min": 0.0,
49
+ "clip_ratio/region_mean": 0.0,
50
+ "completions/clipped_ratio": 0.0625,
51
+ "completions/max_length": 995.5,
52
+ "completions/max_terminated_length": 953.4,
53
+ "completions/mean_length": 511.76875,
54
+ "completions/mean_terminated_length": 480.4899200439453,
55
+ "completions/min_length": 174.1,
56
+ "completions/min_terminated_length": 174.1,
57
+ "entropy": 0.3125879998318851,
58
+ "epoch": 0.009765625,
59
+ "frac_reward_zero_std": 0.0,
60
+ "grad_norm": 0.6785298585891724,
61
+ "learning_rate": 9.2578125e-06,
62
+ "loss": -0.05457568168640137,
63
+ "num_tokens": 759014.0,
64
+ "reward": 0.446875,
65
+ "reward_std": 0.46643308997154237,
66
+ "rewards/reward_func/mean": 0.446875,
67
+ "rewards/reward_func/std": 0.46643308997154237,
68
+ "sampling/importance_sampling_ratio/max": 2.3451719164848326,
69
+ "sampling/importance_sampling_ratio/mean": 0.7033429980278015,
70
+ "sampling/importance_sampling_ratio/min": 0.01897655762732029,
71
+ "sampling/sampling_logp_difference/max": 1.3285840809345246,
72
+ "sampling/sampling_logp_difference/mean": 0.014678731467574835,
73
+ "step": 20,
74
+ "step_time": 42.2913812931627
75
+ },
76
+ {
77
+ "clip_ratio/high_max": 0.0,
78
+ "clip_ratio/high_mean": 0.0,
79
+ "clip_ratio/low_mean": 0.0,
80
+ "clip_ratio/low_min": 0.0,
81
+ "clip_ratio/region_mean": 0.0,
82
+ "completions/clipped_ratio": 0.0875,
83
+ "completions/max_length": 1012.9,
84
+ "completions/max_terminated_length": 909.5,
85
+ "completions/mean_length": 523.709375,
86
+ "completions/mean_terminated_length": 476.59117431640624,
87
+ "completions/min_length": 151.9,
88
+ "completions/min_terminated_length": 151.9,
89
+ "entropy": 0.3126402311958373,
90
+ "epoch": 0.0146484375,
91
+ "frac_reward_zero_std": 0.0,
92
+ "grad_norm": 0.16461844742298126,
93
+ "learning_rate": 8.867187500000001e-06,
94
+ "loss": -0.0469195693731308,
95
+ "num_tokens": 1150177.0,
96
+ "reward": 0.453125,
97
+ "reward_std": 0.4793929249048233,
98
+ "rewards/reward_func/mean": 0.453125,
99
+ "rewards/reward_func/std": 0.4793929249048233,
100
+ "sampling/importance_sampling_ratio/max": 2.0162942886352537,
101
+ "sampling/importance_sampling_ratio/mean": 0.7002942323684692,
102
+ "sampling/importance_sampling_ratio/min": 0.009555302187800407,
103
+ "sampling/sampling_logp_difference/max": 1.7536690354347229,
104
+ "sampling/sampling_logp_difference/mean": 0.014659975934773684,
105
+ "step": 30,
106
+ "step_time": 43.03389266319573
107
+ },
108
+ {
109
+ "clip_ratio/high_max": 0.0,
110
+ "clip_ratio/high_mean": 0.0,
111
+ "clip_ratio/low_mean": 0.0,
112
+ "clip_ratio/low_min": 0.0,
113
+ "clip_ratio/region_mean": 0.0,
114
+ "completions/clipped_ratio": 0.053125,
115
+ "completions/max_length": 1006.0,
116
+ "completions/max_terminated_length": 940.1,
117
+ "completions/mean_length": 507.34375,
118
+ "completions/mean_terminated_length": 479.6489654541016,
119
+ "completions/min_length": 139.0,
120
+ "completions/min_terminated_length": 139.0,
121
+ "entropy": 0.3172651564702392,
122
+ "epoch": 0.01953125,
123
+ "frac_reward_zero_std": 0.0,
124
+ "grad_norm": 0.3692644238471985,
125
+ "learning_rate": 8.4765625e-06,
126
+ "loss": -0.01941492408514023,
127
+ "num_tokens": 1532039.0,
128
+ "reward": 0.5078125,
129
+ "reward_std": 0.4631384491920471,
130
+ "rewards/reward_func/mean": 0.5078125,
131
+ "rewards/reward_func/std": 0.4631384491920471,
132
+ "sampling/importance_sampling_ratio/max": 2.4292322754859925,
133
+ "sampling/importance_sampling_ratio/mean": 0.7138667702674866,
134
+ "sampling/importance_sampling_ratio/min": 0.007732601836323738,
135
+ "sampling/sampling_logp_difference/max": 1.5206358194351197,
136
+ "sampling/sampling_logp_difference/mean": 0.01543022021651268,
137
+ "step": 40,
138
+ "step_time": 42.193549911677835
139
+ },
140
+ {
141
+ "clip_ratio/high_max": 0.0,
142
+ "clip_ratio/high_mean": 0.0,
143
+ "clip_ratio/low_mean": 0.0,
144
+ "clip_ratio/low_min": 0.0,
145
+ "clip_ratio/region_mean": 0.0,
146
+ "completions/clipped_ratio": 0.084375,
147
+ "completions/max_length": 925.6,
148
+ "completions/max_terminated_length": 908.4,
149
+ "completions/mean_length": 523.865625,
150
+ "completions/mean_terminated_length": 484.86688537597655,
151
+ "completions/min_length": 153.8,
152
+ "completions/min_terminated_length": 153.8,
153
+ "entropy": 0.2877701490651816,
154
+ "epoch": 0.0244140625,
155
+ "frac_reward_zero_std": 0.075,
156
+ "grad_norm": 0.13132794201374054,
157
+ "learning_rate": 8.085937500000001e-06,
158
+ "loss": -0.017917218804359435,
159
+ "num_tokens": 1919876.0,
160
+ "reward": 0.6109375,
161
+ "reward_std": 0.45595957338809967,
162
+ "rewards/reward_func/mean": 0.6109375,
163
+ "rewards/reward_func/std": 0.45595957338809967,
164
+ "sampling/importance_sampling_ratio/max": 2.4633763909339903,
165
+ "sampling/importance_sampling_ratio/mean": 0.7146276891231537,
166
+ "sampling/importance_sampling_ratio/min": 0.0,
167
+ "sampling/sampling_logp_difference/max": 1.3434112191200256,
168
+ "sampling/sampling_logp_difference/mean": 0.015156467072665691,
169
+ "step": 50,
170
+ "step_time": 39.661658220365645
171
+ },
172
+ {
173
+ "clip_ratio/high_max": 0.0,
174
+ "clip_ratio/high_mean": 0.0,
175
+ "clip_ratio/low_mean": 0.0,
176
+ "clip_ratio/low_min": 0.0,
177
+ "clip_ratio/region_mean": 0.0,
178
+ "completions/clipped_ratio": 0.05,
179
+ "completions/max_length": 943.1,
180
+ "completions/max_terminated_length": 909.6,
181
+ "completions/mean_length": 510.23125,
182
+ "completions/mean_terminated_length": 484.182958984375,
183
+ "completions/min_length": 155.5,
184
+ "completions/min_terminated_length": 155.5,
185
+ "entropy": 0.27547194906510414,
186
+ "epoch": 0.029296875,
187
+ "frac_reward_zero_std": 0.025,
188
+ "grad_norm": 0.0846150666475296,
189
+ "learning_rate": 7.6953125e-06,
190
+ "loss": -0.035972169041633605,
191
+ "num_tokens": 2305902.0,
192
+ "reward": 0.58125,
193
+ "reward_std": 0.4623547852039337,
194
+ "rewards/reward_func/mean": 0.58125,
195
+ "rewards/reward_func/std": 0.4623547852039337,
196
+ "sampling/importance_sampling_ratio/max": 2.347723662853241,
197
+ "sampling/importance_sampling_ratio/mean": 0.6750797271728516,
198
+ "sampling/importance_sampling_ratio/min": 0.005926066637039184,
199
+ "sampling/sampling_logp_difference/max": 1.4693166017532349,
200
+ "sampling/sampling_logp_difference/mean": 0.01501267571002245,
201
+ "step": 60,
202
+ "step_time": 41.2410288605839
203
+ },
204
+ {
205
+ "clip_ratio/high_max": 0.0,
206
+ "clip_ratio/high_mean": 0.0,
207
+ "clip_ratio/low_mean": 0.0,
208
+ "clip_ratio/low_min": 0.0,
209
+ "clip_ratio/region_mean": 0.0,
210
+ "completions/clipped_ratio": 0.071875,
211
+ "completions/max_length": 979.2,
212
+ "completions/max_terminated_length": 905.5,
213
+ "completions/mean_length": 510.05625,
214
+ "completions/mean_terminated_length": 473.314501953125,
215
+ "completions/min_length": 171.4,
216
+ "completions/min_terminated_length": 171.4,
217
+ "entropy": 0.2838799119926989,
218
+ "epoch": 0.0341796875,
219
+ "frac_reward_zero_std": 0.05,
220
+ "grad_norm": 0.1934366375207901,
221
+ "learning_rate": 7.3046875000000005e-06,
222
+ "loss": -0.06409030556678771,
223
+ "num_tokens": 2690552.0,
224
+ "reward": 0.6546875,
225
+ "reward_std": 0.4583123028278351,
226
+ "rewards/reward_func/mean": 0.6546875,
227
+ "rewards/reward_func/std": 0.4583123028278351,
228
+ "sampling/importance_sampling_ratio/max": 2.5150670051574706,
229
+ "sampling/importance_sampling_ratio/mean": 0.7264489829540253,
230
+ "sampling/importance_sampling_ratio/min": 0.008246464189141989,
231
+ "sampling/sampling_logp_difference/max": 1.62004554271698,
232
+ "sampling/sampling_logp_difference/mean": 0.015931957680732012,
233
+ "step": 70,
234
+ "step_time": 41.73189010284841
235
+ },
236
+ {
237
+ "clip_ratio/high_max": 0.0,
238
+ "clip_ratio/high_mean": 0.0,
239
+ "clip_ratio/low_mean": 0.0,
240
+ "clip_ratio/low_min": 0.0,
241
+ "clip_ratio/region_mean": 0.0,
242
+ "completions/clipped_ratio": 0.078125,
243
+ "completions/max_length": 993.9,
244
+ "completions/max_terminated_length": 949.8,
245
+ "completions/mean_length": 519.25,
246
+ "completions/mean_terminated_length": 477.09031982421874,
247
+ "completions/min_length": 184.2,
248
+ "completions/min_terminated_length": 184.2,
249
+ "entropy": 0.28023276207968595,
250
+ "epoch": 0.0390625,
251
+ "frac_reward_zero_std": 0.025,
252
+ "grad_norm": 0.09890870749950409,
253
+ "learning_rate": 6.9140625e-06,
254
+ "loss": -0.04972947835922241,
255
+ "num_tokens": 3068256.0,
256
+ "reward": 0.659375,
257
+ "reward_std": 0.44355912506580353,
258
+ "rewards/reward_func/mean": 0.659375,
259
+ "rewards/reward_func/std": 0.44355912506580353,
260
+ "sampling/importance_sampling_ratio/max": 2.2669131994247436,
261
+ "sampling/importance_sampling_ratio/mean": 0.6125178694725036,
262
+ "sampling/importance_sampling_ratio/min": 0.007961920648813247,
263
+ "sampling/sampling_logp_difference/max": 1.2455127954483032,
264
+ "sampling/sampling_logp_difference/mean": 0.015829695295542477,
265
+ "step": 80,
266
+ "step_time": 41.7300755944103
267
+ },
268
+ {
269
+ "clip_ratio/high_max": 0.0,
270
+ "clip_ratio/high_mean": 0.0,
271
+ "clip_ratio/low_mean": 0.0,
272
+ "clip_ratio/low_min": 0.0,
273
+ "clip_ratio/region_mean": 0.0,
274
+ "completions/clipped_ratio": 0.1,
275
+ "completions/max_length": 1016.0,
276
+ "completions/max_terminated_length": 936.9,
277
+ "completions/mean_length": 577.265625,
278
+ "completions/mean_terminated_length": 528.5969177246094,
279
+ "completions/min_length": 198.3,
280
+ "completions/min_terminated_length": 198.3,
281
+ "entropy": 0.2634289142210037,
282
+ "epoch": 0.0439453125,
283
+ "frac_reward_zero_std": 0.05,
284
+ "grad_norm": 0.1705273985862732,
285
+ "learning_rate": 6.5234375e-06,
286
+ "loss": -0.08401896357536316,
287
+ "num_tokens": 3485773.0,
288
+ "reward": 0.6875,
289
+ "reward_std": 0.4343861401081085,
290
+ "rewards/reward_func/mean": 0.6875,
291
+ "rewards/reward_func/std": 0.4343861401081085,
292
+ "sampling/importance_sampling_ratio/max": 2.503607487678528,
293
+ "sampling/importance_sampling_ratio/mean": 0.6680708646774292,
294
+ "sampling/importance_sampling_ratio/min": 0.007924840971827506,
295
+ "sampling/sampling_logp_difference/max": 1.9576518058776855,
296
+ "sampling/sampling_logp_difference/mean": 0.015476565062999725,
297
+ "step": 90,
298
+ "step_time": 44.49037704579532
299
+ },
300
+ {
301
+ "clip_ratio/high_max": 0.0,
302
+ "clip_ratio/high_mean": 0.0,
303
+ "clip_ratio/low_mean": 0.0,
304
+ "clip_ratio/low_min": 0.0,
305
+ "clip_ratio/region_mean": 0.0,
306
+ "completions/clipped_ratio": 0.121875,
307
+ "completions/max_length": 1000.4,
308
+ "completions/max_terminated_length": 959.2,
309
+ "completions/mean_length": 580.5625,
310
+ "completions/mean_terminated_length": 524.6508270263672,
311
+ "completions/min_length": 236.8,
312
+ "completions/min_terminated_length": 236.8,
313
+ "entropy": 0.2517876429017633,
314
+ "epoch": 0.048828125,
315
+ "frac_reward_zero_std": 0.15,
316
+ "grad_norm": 0.12978164851665497,
317
+ "learning_rate": 6.132812500000001e-06,
318
+ "loss": -0.06896604895591736,
319
+ "num_tokens": 3896401.0,
320
+ "reward": 0.653125,
321
+ "reward_std": 0.45067343711853025,
322
+ "rewards/reward_func/mean": 0.653125,
323
+ "rewards/reward_func/std": 0.45067343711853025,
324
+ "sampling/importance_sampling_ratio/max": 2.465022587776184,
325
+ "sampling/importance_sampling_ratio/mean": 0.6775680035352707,
326
+ "sampling/importance_sampling_ratio/min": 0.0,
327
+ "sampling/sampling_logp_difference/max": 1.5711746692657471,
328
+ "sampling/sampling_logp_difference/mean": 0.015141925681382418,
329
+ "step": 100,
330
+ "step_time": 43.01227842345834
331
+ },
332
+ {
333
+ "clip_ratio/high_max": 0.0,
334
+ "clip_ratio/high_mean": 0.0,
335
+ "clip_ratio/low_mean": 0.0,
336
+ "clip_ratio/low_min": 0.0,
337
+ "clip_ratio/region_mean": 0.0,
338
+ "completions/clipped_ratio": 0.103125,
339
+ "completions/max_length": 1010.2,
340
+ "completions/max_terminated_length": 945.9,
341
+ "completions/mean_length": 564.890625,
342
+ "completions/mean_terminated_length": 516.3423309326172,
343
+ "completions/min_length": 192.3,
344
+ "completions/min_terminated_length": 192.3,
345
+ "entropy": 0.25614246213808656,
346
+ "epoch": 0.0537109375,
347
+ "frac_reward_zero_std": 0.225,
348
+ "grad_norm": 0.042660851031541824,
349
+ "learning_rate": 5.742187500000001e-06,
350
+ "loss": -0.07317733168601989,
351
+ "num_tokens": 4300918.0,
352
+ "reward": 0.7109375,
353
+ "reward_std": 0.4203103229403496,
354
+ "rewards/reward_func/mean": 0.7109375,
355
+ "rewards/reward_func/std": 0.4203103229403496,
356
+ "sampling/importance_sampling_ratio/max": 2.3434359312057493,
357
+ "sampling/importance_sampling_ratio/mean": 0.6382320910692215,
358
+ "sampling/importance_sampling_ratio/min": 0.006876717309933156,
359
+ "sampling/sampling_logp_difference/max": 1.602103066444397,
360
+ "sampling/sampling_logp_difference/mean": 0.015792049188166856,
361
+ "step": 110,
362
+ "step_time": 43.69292244166136
363
+ },
364
+ {
365
+ "clip_ratio/high_max": 0.0,
366
+ "clip_ratio/high_mean": 0.0,
367
+ "clip_ratio/low_mean": 0.0,
368
+ "clip_ratio/low_min": 0.0,
369
+ "clip_ratio/region_mean": 0.0,
370
+ "completions/clipped_ratio": 0.040625,
371
+ "completions/max_length": 912.5,
372
+ "completions/max_terminated_length": 867.4,
373
+ "completions/mean_length": 467.778125,
374
+ "completions/mean_terminated_length": 447.03748779296876,
375
+ "completions/min_length": 190.6,
376
+ "completions/min_terminated_length": 190.6,
377
+ "entropy": 0.28152717975899577,
378
+ "epoch": 0.05859375,
379
+ "frac_reward_zero_std": 0.225,
380
+ "grad_norm": 0.1590559482574463,
381
+ "learning_rate": 5.3515625000000005e-06,
382
+ "loss": -0.016551096737384797,
383
+ "num_tokens": 4647967.0,
384
+ "reward": 0.7625,
385
+ "reward_std": 0.4036805719137192,
386
+ "rewards/reward_func/mean": 0.7625,
387
+ "rewards/reward_func/std": 0.4036805719137192,
388
+ "sampling/importance_sampling_ratio/max": 2.325450301170349,
389
+ "sampling/importance_sampling_ratio/mean": 0.558803403377533,
390
+ "sampling/importance_sampling_ratio/min": 0.009663067432120443,
391
+ "sampling/sampling_logp_difference/max": 1.6965322017669677,
392
+ "sampling/sampling_logp_difference/mean": 0.017247257009148597,
393
+ "step": 120,
394
+ "step_time": 37.80693711265921
395
+ },
396
+ {
397
+ "clip_ratio/high_max": 0.0,
398
+ "clip_ratio/high_mean": 0.0,
399
+ "clip_ratio/low_mean": 0.0,
400
+ "clip_ratio/low_min": 0.0,
401
+ "clip_ratio/region_mean": 0.0,
402
+ "completions/clipped_ratio": 0.059375,
403
+ "completions/max_length": 965.7,
404
+ "completions/max_terminated_length": 933.1,
405
+ "completions/mean_length": 500.625,
406
+ "completions/mean_terminated_length": 468.8048461914062,
407
+ "completions/min_length": 182.9,
408
+ "completions/min_terminated_length": 182.9,
409
+ "entropy": 0.26958370129577813,
410
+ "epoch": 0.0634765625,
411
+ "frac_reward_zero_std": 0.125,
412
+ "grad_norm": 1.2840863466262817,
413
+ "learning_rate": 4.9609375e-06,
414
+ "loss": -0.02979734241962433,
415
+ "num_tokens": 5017103.0,
416
+ "reward": 0.728125,
417
+ "reward_std": 0.4005833774805069,
418
+ "rewards/reward_func/mean": 0.728125,
419
+ "rewards/reward_func/std": 0.4005833774805069,
420
+ "sampling/importance_sampling_ratio/max": 2.274832272529602,
421
+ "sampling/importance_sampling_ratio/mean": 0.6487800061702729,
422
+ "sampling/importance_sampling_ratio/min": 0.0036865059286355973,
423
+ "sampling/sampling_logp_difference/max": 1.5818899750709534,
424
+ "sampling/sampling_logp_difference/mean": 0.016566928662359714,
425
+ "step": 130,
426
+ "step_time": 41.61081804931164
427
+ },
428
+ {
429
+ "clip_ratio/high_max": 0.0,
430
+ "clip_ratio/high_mean": 0.0,
431
+ "clip_ratio/low_mean": 0.0,
432
+ "clip_ratio/low_min": 0.0,
433
+ "clip_ratio/region_mean": 0.0,
434
+ "completions/clipped_ratio": 0.053125,
435
+ "completions/max_length": 941.2,
436
+ "completions/max_terminated_length": 909.3,
437
+ "completions/mean_length": 509.8375,
438
+ "completions/mean_terminated_length": 483.2834991455078,
439
+ "completions/min_length": 185.4,
440
+ "completions/min_terminated_length": 185.4,
441
+ "entropy": 0.262696215743199,
442
+ "epoch": 0.068359375,
443
+ "frac_reward_zero_std": 0.175,
444
+ "grad_norm": 0.1525411456823349,
445
+ "learning_rate": 4.5703125e-06,
446
+ "loss": -0.046766582131385806,
447
+ "num_tokens": 5398483.0,
448
+ "reward": 0.75625,
449
+ "reward_std": 0.39561293721199037,
450
+ "rewards/reward_func/mean": 0.75625,
451
+ "rewards/reward_func/std": 0.39561293721199037,
452
+ "sampling/importance_sampling_ratio/max": 2.4634074449539183,
453
+ "sampling/importance_sampling_ratio/mean": 0.5994631499052048,
454
+ "sampling/importance_sampling_ratio/min": 0.0038327477872371674,
455
+ "sampling/sampling_logp_difference/max": 1.8424236536026002,
456
+ "sampling/sampling_logp_difference/mean": 0.01626686742529273,
457
+ "step": 140,
458
+ "step_time": 40.34408489987254
459
+ },
460
+ {
461
+ "clip_ratio/high_max": 0.0,
462
+ "clip_ratio/high_mean": 0.0,
463
+ "clip_ratio/low_mean": 0.0,
464
+ "clip_ratio/low_min": 0.0,
465
+ "clip_ratio/region_mean": 0.0,
466
+ "completions/clipped_ratio": 0.071875,
467
+ "completions/max_length": 988.1,
468
+ "completions/max_terminated_length": 872.2,
469
+ "completions/mean_length": 511.871875,
470
+ "completions/mean_terminated_length": 478.7667999267578,
471
+ "completions/min_length": 211.1,
472
+ "completions/min_terminated_length": 211.1,
473
+ "entropy": 0.252231373032555,
474
+ "epoch": 0.0732421875,
475
+ "frac_reward_zero_std": 0.1,
476
+ "grad_norm": 0.27685225009918213,
477
+ "learning_rate": 4.1796875e-06,
478
+ "loss": -0.051500356197357176,
479
+ "num_tokens": 5784674.0,
480
+ "reward": 0.7015625,
481
+ "reward_std": 0.4280380368232727,
482
+ "rewards/reward_func/mean": 0.7015625,
483
+ "rewards/reward_func/std": 0.4280380368232727,
484
+ "sampling/importance_sampling_ratio/max": 2.3887526750564576,
485
+ "sampling/importance_sampling_ratio/mean": 0.642585214972496,
486
+ "sampling/importance_sampling_ratio/min": 0.00046140169724822043,
487
+ "sampling/sampling_logp_difference/max": 1.4521429300308228,
488
+ "sampling/sampling_logp_difference/mean": 0.016321948915719985,
489
+ "step": 150,
490
+ "step_time": 42.41349179409444
491
+ },
492
+ {
493
+ "clip_ratio/high_max": 0.0,
494
+ "clip_ratio/high_mean": 0.0,
495
+ "clip_ratio/low_mean": 0.0,
496
+ "clip_ratio/low_min": 0.0,
497
+ "clip_ratio/region_mean": 0.0,
498
+ "completions/clipped_ratio": 0.065625,
499
+ "completions/max_length": 978.6,
500
+ "completions/max_terminated_length": 931.7,
501
+ "completions/mean_length": 522.546875,
502
+ "completions/mean_terminated_length": 488.7979339599609,
503
+ "completions/min_length": 188.4,
504
+ "completions/min_terminated_length": 188.4,
505
+ "entropy": 0.24772414807230234,
506
+ "epoch": 0.078125,
507
+ "frac_reward_zero_std": 0.15,
508
+ "grad_norm": 0.0861799418926239,
509
+ "learning_rate": 3.7890625e-06,
510
+ "loss": -0.0425748199224472,
511
+ "num_tokens": 6171721.0,
512
+ "reward": 0.7328125,
513
+ "reward_std": 0.4073425203561783,
514
+ "rewards/reward_func/mean": 0.7328125,
515
+ "rewards/reward_func/std": 0.4073425203561783,
516
+ "sampling/importance_sampling_ratio/max": 2.409035348892212,
517
+ "sampling/importance_sampling_ratio/mean": 0.580704128742218,
518
+ "sampling/importance_sampling_ratio/min": 0.002471921592950821,
519
+ "sampling/sampling_logp_difference/max": 1.5051016092300415,
520
+ "sampling/sampling_logp_difference/mean": 0.016132805589586498,
521
+ "step": 160,
522
+ "step_time": 41.94835491515696
523
+ },
524
+ {
525
+ "clip_ratio/high_max": 0.0,
526
+ "clip_ratio/high_mean": 0.0,
527
+ "clip_ratio/low_mean": 0.0,
528
+ "clip_ratio/low_min": 0.0,
529
+ "clip_ratio/region_mean": 0.0,
530
+ "completions/clipped_ratio": 0.065625,
531
+ "completions/max_length": 991.1,
532
+ "completions/max_terminated_length": 918.9,
533
+ "completions/mean_length": 528.475,
534
+ "completions/mean_terminated_length": 496.8511444091797,
535
+ "completions/min_length": 192.1,
536
+ "completions/min_terminated_length": 192.1,
537
+ "entropy": 0.23968204399570822,
538
+ "epoch": 0.0830078125,
539
+ "frac_reward_zero_std": 0.25,
540
+ "grad_norm": 0.042260002344846725,
541
+ "learning_rate": 3.3984375000000004e-06,
542
+ "loss": -0.03924897015094757,
543
+ "num_tokens": 6564921.0,
544
+ "reward": 0.7671875,
545
+ "reward_std": 0.3848088890314102,
546
+ "rewards/reward_func/mean": 0.7671875,
547
+ "rewards/reward_func/std": 0.3848088890314102,
548
+ "sampling/importance_sampling_ratio/max": 2.4140881180763243,
549
+ "sampling/importance_sampling_ratio/mean": 0.6277039766311645,
550
+ "sampling/importance_sampling_ratio/min": 0.010604271292686462,
551
+ "sampling/sampling_logp_difference/max": 2.077266240119934,
552
+ "sampling/sampling_logp_difference/mean": 0.015553839039057493,
553
+ "step": 170,
554
+ "step_time": 42.652561436221006
555
+ },
556
+ {
557
+ "clip_ratio/high_max": 0.0,
558
+ "clip_ratio/high_mean": 0.0,
559
+ "clip_ratio/low_mean": 0.0,
560
+ "clip_ratio/low_min": 0.0,
561
+ "clip_ratio/region_mean": 0.0,
562
+ "completions/clipped_ratio": 0.046875,
563
+ "completions/max_length": 981.2,
564
+ "completions/max_terminated_length": 853.4,
565
+ "completions/mean_length": 505.45,
566
+ "completions/mean_terminated_length": 480.1022125244141,
567
+ "completions/min_length": 191.8,
568
+ "completions/min_terminated_length": 191.8,
569
+ "entropy": 0.24593189600855112,
570
+ "epoch": 0.087890625,
571
+ "frac_reward_zero_std": 0.075,
572
+ "grad_norm": 0.06853368878364563,
573
+ "learning_rate": 3.0078125000000003e-06,
574
+ "loss": -0.04325226545333862,
575
+ "num_tokens": 6947625.0,
576
+ "reward": 0.75,
577
+ "reward_std": 0.40317397117614745,
578
+ "rewards/reward_func/mean": 0.75,
579
+ "rewards/reward_func/std": 0.40317397117614745,
580
+ "sampling/importance_sampling_ratio/max": 2.4479388117790224,
581
+ "sampling/importance_sampling_ratio/mean": 0.5681718856096267,
582
+ "sampling/importance_sampling_ratio/min": 0.000738889304921031,
583
+ "sampling/sampling_logp_difference/max": 1.9939045190811158,
584
+ "sampling/sampling_logp_difference/mean": 0.016909655183553696,
585
+ "step": 180,
586
+ "step_time": 41.504886724427344
587
+ },
588
+ {
589
+ "clip_ratio/high_max": 0.0,
590
+ "clip_ratio/high_mean": 0.0,
591
+ "clip_ratio/low_mean": 0.0,
592
+ "clip_ratio/low_min": 0.0,
593
+ "clip_ratio/region_mean": 0.0,
594
+ "completions/clipped_ratio": 0.05625,
595
+ "completions/max_length": 998.6,
596
+ "completions/max_terminated_length": 907.2,
597
+ "completions/mean_length": 512.33125,
598
+ "completions/mean_terminated_length": 482.89744873046874,
599
+ "completions/min_length": 189.9,
600
+ "completions/min_terminated_length": 189.9,
601
+ "entropy": 0.24478109064511955,
602
+ "epoch": 0.0927734375,
603
+ "frac_reward_zero_std": 0.05,
604
+ "grad_norm": 0.08906801044940948,
605
+ "learning_rate": 2.6171875e-06,
606
+ "loss": -0.043896269798278806,
607
+ "num_tokens": 7338203.0,
608
+ "reward": 0.7109375,
609
+ "reward_std": 0.4192426443099976,
610
+ "rewards/reward_func/mean": 0.7109375,
611
+ "rewards/reward_func/std": 0.4192426443099976,
612
+ "sampling/importance_sampling_ratio/max": 2.4853260040283205,
613
+ "sampling/importance_sampling_ratio/mean": 0.6375262677669525,
614
+ "sampling/importance_sampling_ratio/min": 0.0017541885375976563,
615
+ "sampling/sampling_logp_difference/max": 2.002353346347809,
616
+ "sampling/sampling_logp_difference/mean": 0.016874660179018973,
617
+ "step": 190,
618
+ "step_time": 43.31960063055158
619
+ },
620
+ {
621
+ "clip_ratio/high_max": 0.0,
622
+ "clip_ratio/high_mean": 0.0,
623
+ "clip_ratio/low_mean": 0.0,
624
+ "clip_ratio/low_min": 0.0,
625
+ "clip_ratio/region_mean": 0.0,
626
+ "completions/clipped_ratio": 0.065625,
627
+ "completions/max_length": 998.6,
628
+ "completions/max_terminated_length": 920.4,
629
+ "completions/mean_length": 523.025,
630
+ "completions/mean_terminated_length": 488.0899169921875,
631
+ "completions/min_length": 203.3,
632
+ "completions/min_terminated_length": 203.3,
633
+ "entropy": 0.2409972614143044,
634
+ "epoch": 0.09765625,
635
+ "frac_reward_zero_std": 0.2,
636
+ "grad_norm": 0.08618272840976715,
637
+ "learning_rate": 2.2265625000000003e-06,
638
+ "loss": -0.05364810824394226,
639
+ "num_tokens": 7733595.0,
640
+ "reward": 0.7578125,
641
+ "reward_std": 0.40019345581531524,
642
+ "rewards/reward_func/mean": 0.7578125,
643
+ "rewards/reward_func/std": 0.40019345581531524,
644
+ "sampling/importance_sampling_ratio/max": 2.357998478412628,
645
+ "sampling/importance_sampling_ratio/mean": 0.5750122100114823,
646
+ "sampling/importance_sampling_ratio/min": 0.00275895893573761,
647
+ "sampling/sampling_logp_difference/max": 1.6211442470550537,
648
+ "sampling/sampling_logp_difference/mean": 0.016668787319213153,
649
+ "step": 200,
650
+ "step_time": 43.26172666847706
651
+ },
652
+ {
653
+ "clip_ratio/high_max": 0.0,
654
+ "clip_ratio/high_mean": 0.0,
655
+ "clip_ratio/low_mean": 0.0,
656
+ "clip_ratio/low_min": 0.0,
657
+ "clip_ratio/region_mean": 0.0,
658
+ "completions/clipped_ratio": 0.05,
659
+ "completions/max_length": 999.6,
660
+ "completions/max_terminated_length": 929.4,
661
+ "completions/mean_length": 511.834375,
662
+ "completions/mean_terminated_length": 485.07744750976565,
663
+ "completions/min_length": 194.9,
664
+ "completions/min_terminated_length": 194.9,
665
+ "entropy": 0.2472895231563598,
666
+ "epoch": 0.1025390625,
667
+ "frac_reward_zero_std": 0.075,
668
+ "grad_norm": 0.04566340520977974,
669
+ "learning_rate": 1.8359375000000002e-06,
670
+ "loss": -0.031854945421218875,
671
+ "num_tokens": 8117582.0,
672
+ "reward": 0.7171875,
673
+ "reward_std": 0.4062408864498138,
674
+ "rewards/reward_func/mean": 0.7171875,
675
+ "rewards/reward_func/std": 0.4062408864498138,
676
+ "sampling/importance_sampling_ratio/max": 2.44709974527359,
677
+ "sampling/importance_sampling_ratio/mean": 0.5207547932863236,
678
+ "sampling/importance_sampling_ratio/min": 0.0020654477179050446,
679
+ "sampling/sampling_logp_difference/max": 2.7428050518035887,
680
+ "sampling/sampling_logp_difference/mean": 0.017336908262223005,
681
+ "step": 210,
682
+ "step_time": 43.34906991906464
683
+ },
684
+ {
685
+ "clip_ratio/high_max": 0.0,
686
+ "clip_ratio/high_mean": 0.0,
687
+ "clip_ratio/low_mean": 0.0,
688
+ "clip_ratio/low_min": 0.0,
689
+ "clip_ratio/region_mean": 0.0,
690
+ "completions/clipped_ratio": 0.05625,
691
+ "completions/max_length": 1010.2,
692
+ "completions/max_terminated_length": 981.6,
693
+ "completions/mean_length": 559.628125,
694
+ "completions/mean_terminated_length": 534.4404724121093,
695
+ "completions/min_length": 220.5,
696
+ "completions/min_terminated_length": 220.5,
697
+ "entropy": 0.22819670476019382,
698
+ "epoch": 0.107421875,
699
+ "frac_reward_zero_std": 0.225,
700
+ "grad_norm": 0.07086535543203354,
701
+ "learning_rate": 1.4453125000000002e-06,
702
+ "loss": -0.04208610653877258,
703
+ "num_tokens": 8532735.0,
704
+ "reward": 0.803125,
705
+ "reward_std": 0.3462395280599594,
706
+ "rewards/reward_func/mean": 0.803125,
707
+ "rewards/reward_func/std": 0.3462395280599594,
708
+ "sampling/importance_sampling_ratio/max": 2.328373467922211,
709
+ "sampling/importance_sampling_ratio/mean": 0.5397515326738358,
710
+ "sampling/importance_sampling_ratio/min": 0.0012410758063197136,
711
+ "sampling/sampling_logp_difference/max": 2.075514018535614,
712
+ "sampling/sampling_logp_difference/mean": 0.015707963891327382,
713
+ "step": 220,
714
+ "step_time": 43.9373246755451
715
+ },
716
+ {
717
+ "clip_ratio/high_max": 0.0,
718
+ "clip_ratio/high_mean": 0.0,
719
+ "clip_ratio/low_mean": 0.0,
720
+ "clip_ratio/low_min": 0.0,
721
+ "clip_ratio/region_mean": 0.0,
722
+ "completions/clipped_ratio": 0.025,
723
+ "completions/max_length": 934.4,
724
+ "completions/max_terminated_length": 865.3,
725
+ "completions/mean_length": 459.71875,
726
+ "completions/mean_terminated_length": 446.5812713623047,
727
+ "completions/min_length": 199.7,
728
+ "completions/min_terminated_length": 199.7,
729
+ "entropy": 0.26363498624414206,
730
+ "epoch": 0.1123046875,
731
+ "frac_reward_zero_std": 0.225,
732
+ "grad_norm": 0.04578734561800957,
733
+ "learning_rate": 1.0546875e-06,
734
+ "loss": -0.03878411948680878,
735
+ "num_tokens": 8879581.0,
736
+ "reward": 0.771875,
737
+ "reward_std": 0.39926943778991697,
738
+ "rewards/reward_func/mean": 0.771875,
739
+ "rewards/reward_func/std": 0.39926943778991697,
740
+ "sampling/importance_sampling_ratio/max": 2.2447435021400453,
741
+ "sampling/importance_sampling_ratio/mean": 0.5411602407693863,
742
+ "sampling/importance_sampling_ratio/min": 0.0,
743
+ "sampling/sampling_logp_difference/max": 2.0481242179870605,
744
+ "sampling/sampling_logp_difference/mean": 0.0182870015501976,
745
+ "step": 230,
746
+ "step_time": 39.24789510220289
747
+ },
748
+ {
749
+ "clip_ratio/high_max": 0.0,
750
+ "clip_ratio/high_mean": 0.0,
751
+ "clip_ratio/low_mean": 0.0,
752
+ "clip_ratio/low_min": 0.0,
753
+ "clip_ratio/region_mean": 0.0,
754
+ "completions/clipped_ratio": 0.065625,
755
+ "completions/max_length": 1015.1,
756
+ "completions/max_terminated_length": 915.1,
757
+ "completions/mean_length": 540.621875,
758
+ "completions/mean_terminated_length": 508.8499755859375,
759
+ "completions/min_length": 190.4,
760
+ "completions/min_terminated_length": 190.4,
761
+ "entropy": 0.21832518568262457,
762
+ "epoch": 0.1171875,
763
+ "frac_reward_zero_std": 0.1,
764
+ "grad_norm": 0.13058459758758545,
765
+ "learning_rate": 6.640625e-07,
766
+ "loss": -0.06161265373229981,
767
+ "num_tokens": 9287260.0,
768
+ "reward": 0.73125,
769
+ "reward_std": 0.41004206240177155,
770
+ "rewards/reward_func/mean": 0.73125,
771
+ "rewards/reward_func/std": 0.41004206240177155,
772
+ "sampling/importance_sampling_ratio/max": 2.472160220146179,
773
+ "sampling/importance_sampling_ratio/mean": 0.5784197092056275,
774
+ "sampling/importance_sampling_ratio/min": 0.0,
775
+ "sampling/sampling_logp_difference/max": 1.7262341976165771,
776
+ "sampling/sampling_logp_difference/mean": 0.015848540980368853,
777
+ "step": 240,
778
+ "step_time": 43.117547643184665
779
+ },
780
+ {
781
+ "clip_ratio/high_max": 0.0,
782
+ "clip_ratio/high_mean": 0.0,
783
+ "clip_ratio/low_mean": 0.0,
784
+ "clip_ratio/low_min": 0.0,
785
+ "clip_ratio/region_mean": 0.0,
786
+ "completions/clipped_ratio": 0.04375,
787
+ "completions/max_length": 993.0,
788
+ "completions/max_terminated_length": 960.8,
789
+ "completions/mean_length": 500.878125,
790
+ "completions/mean_terminated_length": 477.1601531982422,
791
+ "completions/min_length": 174.9,
792
+ "completions/min_terminated_length": 174.9,
793
+ "entropy": 0.2411346232984215,
794
+ "epoch": 0.1220703125,
795
+ "frac_reward_zero_std": 0.225,
796
+ "grad_norm": 0.11726124584674835,
797
+ "learning_rate": 2.7343750000000003e-07,
798
+ "loss": -0.03523572683334351,
799
+ "num_tokens": 9664277.0,
800
+ "reward": 0.7765625,
801
+ "reward_std": 0.38842655122280123,
802
+ "rewards/reward_func/mean": 0.7765625,
803
+ "rewards/reward_func/std": 0.38842655122280123,
804
+ "sampling/importance_sampling_ratio/max": 2.381782817840576,
805
+ "sampling/importance_sampling_ratio/mean": 0.6045764774084091,
806
+ "sampling/importance_sampling_ratio/min": 0.0,
807
+ "sampling/sampling_logp_difference/max": 1.8437294721603394,
808
+ "sampling/sampling_logp_difference/mean": 0.016771893482655287,
809
+ "step": 250,
810
+ "step_time": 42.89663612283766
811
+ }
812
+ ],
813
+ "logging_steps": 10,
814
+ "max_steps": 256,
815
+ "num_input_tokens_seen": 9920125,
816
+ "num_train_epochs": 1,
817
+ "save_steps": 257,
818
+ "stateful_callbacks": {
819
+ "TrainerControl": {
820
+ "args": {
821
+ "should_epoch_stop": false,
822
+ "should_evaluate": false,
823
+ "should_log": false,
824
+ "should_save": true,
825
+ "should_training_stop": true
826
+ },
827
+ "attributes": {}
828
+ }
829
+ },
830
+ "total_flos": 0.0,
831
+ "train_batch_size": 2,
832
+ "trial_name": null,
833
+ "trial_params": null
834
+ }
rl_grpo/charter_real_4x_thinking/checkpoint-32/adapter_config.json ADDED
@@ -0,0 +1,191 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alora_invocation_tokens": null,
3
+ "alpha_pattern": {},
4
+ "arrow_config": null,
5
+ "auto_mapping": null,
6
+ "base_model_name_or_path": "/workspace/rl3t_a/parent",
7
+ "bias": "none",
8
+ "corda_config": null,
9
+ "ensure_weight_tying": false,
10
+ "eva_config": null,
11
+ "exclude_modules": null,
12
+ "fan_in_fan_out": false,
13
+ "inference_mode": true,
14
+ "init_lora_weights": true,
15
+ "layer_replication": null,
16
+ "layers_pattern": null,
17
+ "layers_to_transform": null,
18
+ "loftq_config": {},
19
+ "lora_alpha": 64,
20
+ "lora_bias": false,
21
+ "lora_dropout": 0.0,
22
+ "lora_ga_config": null,
23
+ "megatron_config": null,
24
+ "megatron_core": "megatron.core",
25
+ "modules_to_save": null,
26
+ "monteclora_config": null,
27
+ "peft_type": "LORA",
28
+ "peft_version": "0.20.0",
29
+ "qalora_group_size": 16,
30
+ "r": 32,
31
+ "rank_pattern": {},
32
+ "revision": null,
33
+ "target_modules": [
34
+ "41.self_attn.k_proj",
35
+ "language_model.layers.6.self_attn.q_proj",
36
+ "language_model.layers.22.self_attn.v_proj",
37
+ "45.self_attn.q_proj",
38
+ "34.self_attn.v_proj",
39
+ "32.self_attn.v_proj",
40
+ "language_model.layers.1.self_attn.k_proj",
41
+ "46.self_attn.k_proj",
42
+ "40.self_attn.q_proj",
43
+ "language_model.layers.18.self_attn.v_proj",
44
+ "31.self_attn.k_proj",
45
+ "38.self_attn.q_proj",
46
+ "language_model.layers.7.self_attn.q_proj",
47
+ "31.self_attn.q_proj",
48
+ "language_model.layers.21.self_attn.v_proj",
49
+ "up_proj",
50
+ "36.self_attn.k_proj",
51
+ "language_model.layers.26.self_attn.v_proj",
52
+ "language_model.layers.9.self_attn.k_proj",
53
+ "language_model.layers.12.self_attn.v_proj",
54
+ "language_model.layers.10.self_attn.q_proj",
55
+ "29.self_attn.q_proj",
56
+ "35.self_attn.q_proj",
57
+ "language_model.layers.15.self_attn.k_proj",
58
+ "40.self_attn.v_proj",
59
+ "language_model.layers.9.self_attn.v_proj",
60
+ "language_model.layers.11.self_attn.q_proj",
61
+ "language_model.layers.25.self_attn.q_proj",
62
+ "language_model.layers.7.self_attn.v_proj",
63
+ "35.self_attn.v_proj",
64
+ "44.self_attn.v_proj",
65
+ "o_proj",
66
+ "47.self_attn.v_proj",
67
+ "language_model.layers.6.self_attn.v_proj",
68
+ "language_model.layers.1.self_attn.v_proj",
69
+ "35.self_attn.k_proj",
70
+ "language_model.layers.17.self_attn.v_proj",
71
+ "language_model.layers.8.self_attn.k_proj",
72
+ "language_model.layers.21.self_attn.q_proj",
73
+ "39.self_attn.k_proj",
74
+ "language_model.layers.12.self_attn.k_proj",
75
+ "language_model.layers.14.self_attn.q_proj",
76
+ "language_model.layers.11.self_attn.v_proj",
77
+ "34.self_attn.k_proj",
78
+ "language_model.layers.10.self_attn.k_proj",
79
+ "language_model.layers.5.self_attn.k_proj",
80
+ "language_model.layers.17.self_attn.q_proj",
81
+ "44.self_attn.k_proj",
82
+ "language_model.layers.20.self_attn.v_proj",
83
+ "30.self_attn.v_proj",
84
+ "29.self_attn.v_proj",
85
+ "language_model.layers.4.self_attn.q_proj",
86
+ "language_model.layers.9.self_attn.q_proj",
87
+ "language_model.layers.4.self_attn.v_proj",
88
+ "27.self_attn.k_proj",
89
+ "38.self_attn.v_proj",
90
+ "language_model.layers.18.self_attn.k_proj",
91
+ "down_proj",
92
+ "45.self_attn.v_proj",
93
+ "language_model.layers.26.self_attn.q_proj",
94
+ "language_model.layers.2.self_attn.q_proj",
95
+ "language_model.layers.3.self_attn.q_proj",
96
+ "language_model.layers.2.self_attn.v_proj",
97
+ "language_model.layers.16.self_attn.q_proj",
98
+ "language_model.layers.2.self_attn.k_proj",
99
+ "27.self_attn.v_proj",
100
+ "33.self_attn.q_proj",
101
+ "43.self_attn.q_proj",
102
+ "31.self_attn.v_proj",
103
+ "33.self_attn.v_proj",
104
+ "42.self_attn.q_proj",
105
+ "28.self_attn.k_proj",
106
+ "language_model.layers.13.self_attn.v_proj",
107
+ "46.self_attn.v_proj",
108
+ "language_model.layers.19.self_attn.k_proj",
109
+ "language_model.layers.0.self_attn.q_proj",
110
+ "language_model.layers.3.self_attn.k_proj",
111
+ "language_model.layers.14.self_attn.v_proj",
112
+ "37.self_attn.v_proj",
113
+ "43.self_attn.v_proj",
114
+ "39.self_attn.q_proj",
115
+ "language_model.layers.5.self_attn.q_proj",
116
+ "language_model.layers.24.self_attn.q_proj",
117
+ "language_model.layers.24.self_attn.k_proj",
118
+ "47.self_attn.q_proj",
119
+ "30.self_attn.q_proj",
120
+ "language_model.layers.4.self_attn.k_proj",
121
+ "33.self_attn.k_proj",
122
+ "40.self_attn.k_proj",
123
+ "language_model.layers.10.self_attn.v_proj",
124
+ "language_model.layers.25.self_attn.v_proj",
125
+ "37.self_attn.k_proj",
126
+ "language_model.layers.19.self_attn.v_proj",
127
+ "41.self_attn.q_proj",
128
+ "language_model.layers.22.self_attn.k_proj",
129
+ "46.self_attn.q_proj",
130
+ "28.self_attn.q_proj",
131
+ "language_model.layers.7.self_attn.k_proj",
132
+ "language_model.layers.21.self_attn.k_proj",
133
+ "language_model.layers.6.self_attn.k_proj",
134
+ "28.self_attn.v_proj",
135
+ "language_model.layers.3.self_attn.v_proj",
136
+ "36.self_attn.q_proj",
137
+ "language_model.layers.23.self_attn.k_proj",
138
+ "language_model.layers.0.self_attn.v_proj",
139
+ "language_model.layers.15.self_attn.v_proj",
140
+ "language_model.layers.15.self_attn.q_proj",
141
+ "gate_proj",
142
+ "34.self_attn.q_proj",
143
+ "language_model.layers.18.self_attn.q_proj",
144
+ "43.self_attn.k_proj",
145
+ "language_model.layers.11.self_attn.k_proj",
146
+ "language_model.layers.22.self_attn.q_proj",
147
+ "language_model.layers.17.self_attn.k_proj",
148
+ "36.self_attn.v_proj",
149
+ "language_model.layers.26.self_attn.k_proj",
150
+ "45.self_attn.k_proj",
151
+ "language_model.layers.12.self_attn.q_proj",
152
+ "37.self_attn.q_proj",
153
+ "44.self_attn.q_proj",
154
+ "language_model.layers.14.self_attn.k_proj",
155
+ "language_model.layers.23.self_attn.v_proj",
156
+ "42.self_attn.v_proj",
157
+ "47.self_attn.k_proj",
158
+ "language_model.layers.20.self_attn.q_proj",
159
+ "language_model.layers.5.self_attn.v_proj",
160
+ "language_model.layers.20.self_attn.k_proj",
161
+ "language_model.layers.24.self_attn.v_proj",
162
+ "32.self_attn.q_proj",
163
+ "language_model.layers.13.self_attn.q_proj",
164
+ "30.self_attn.k_proj",
165
+ "39.self_attn.v_proj",
166
+ "32.self_attn.k_proj",
167
+ "language_model.layers.1.self_attn.q_proj",
168
+ "language_model.layers.0.self_attn.k_proj",
169
+ "language_model.layers.19.self_attn.q_proj",
170
+ "27.self_attn.q_proj",
171
+ "language_model.layers.8.self_attn.q_proj",
172
+ "language_model.layers.25.self_attn.k_proj",
173
+ "language_model.layers.8.self_attn.v_proj",
174
+ "language_model.layers.16.self_attn.k_proj",
175
+ "38.self_attn.k_proj",
176
+ "language_model.layers.16.self_attn.v_proj",
177
+ "29.self_attn.k_proj",
178
+ "language_model.layers.13.self_attn.k_proj",
179
+ "41.self_attn.v_proj",
180
+ "language_model.layers.23.self_attn.q_proj",
181
+ "42.self_attn.k_proj"
182
+ ],
183
+ "target_parameters": null,
184
+ "task_type": "CAUSAL_LM",
185
+ "trainable_token_indices": null,
186
+ "use_bdlora": null,
187
+ "use_dora": false,
188
+ "use_qalora": false,
189
+ "use_rslora": false,
190
+ "velora_config": null
191
+ }
rl_grpo/charter_real_4x_thinking/checkpoint-32/adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2ce39350cbf59242091259c35d21856ac7961a1ae2aca5ae7ac22606fca3bf79
3
+ size 523864568
rl_grpo/charter_real_4x_thinking/checkpoint-64/adapter_config.json ADDED
@@ -0,0 +1,191 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alora_invocation_tokens": null,
3
+ "alpha_pattern": {},
4
+ "arrow_config": null,
5
+ "auto_mapping": null,
6
+ "base_model_name_or_path": "/workspace/rl3t_a/parent",
7
+ "bias": "none",
8
+ "corda_config": null,
9
+ "ensure_weight_tying": false,
10
+ "eva_config": null,
11
+ "exclude_modules": null,
12
+ "fan_in_fan_out": false,
13
+ "inference_mode": true,
14
+ "init_lora_weights": true,
15
+ "layer_replication": null,
16
+ "layers_pattern": null,
17
+ "layers_to_transform": null,
18
+ "loftq_config": {},
19
+ "lora_alpha": 64,
20
+ "lora_bias": false,
21
+ "lora_dropout": 0.0,
22
+ "lora_ga_config": null,
23
+ "megatron_config": null,
24
+ "megatron_core": "megatron.core",
25
+ "modules_to_save": null,
26
+ "monteclora_config": null,
27
+ "peft_type": "LORA",
28
+ "peft_version": "0.20.0",
29
+ "qalora_group_size": 16,
30
+ "r": 32,
31
+ "rank_pattern": {},
32
+ "revision": null,
33
+ "target_modules": [
34
+ "41.self_attn.k_proj",
35
+ "language_model.layers.6.self_attn.q_proj",
36
+ "language_model.layers.22.self_attn.v_proj",
37
+ "45.self_attn.q_proj",
38
+ "34.self_attn.v_proj",
39
+ "32.self_attn.v_proj",
40
+ "language_model.layers.1.self_attn.k_proj",
41
+ "46.self_attn.k_proj",
42
+ "40.self_attn.q_proj",
43
+ "language_model.layers.18.self_attn.v_proj",
44
+ "31.self_attn.k_proj",
45
+ "38.self_attn.q_proj",
46
+ "language_model.layers.7.self_attn.q_proj",
47
+ "31.self_attn.q_proj",
48
+ "language_model.layers.21.self_attn.v_proj",
49
+ "up_proj",
50
+ "36.self_attn.k_proj",
51
+ "language_model.layers.26.self_attn.v_proj",
52
+ "language_model.layers.9.self_attn.k_proj",
53
+ "language_model.layers.12.self_attn.v_proj",
54
+ "language_model.layers.10.self_attn.q_proj",
55
+ "29.self_attn.q_proj",
56
+ "35.self_attn.q_proj",
57
+ "language_model.layers.15.self_attn.k_proj",
58
+ "40.self_attn.v_proj",
59
+ "language_model.layers.9.self_attn.v_proj",
60
+ "language_model.layers.11.self_attn.q_proj",
61
+ "language_model.layers.25.self_attn.q_proj",
62
+ "language_model.layers.7.self_attn.v_proj",
63
+ "35.self_attn.v_proj",
64
+ "44.self_attn.v_proj",
65
+ "o_proj",
66
+ "47.self_attn.v_proj",
67
+ "language_model.layers.6.self_attn.v_proj",
68
+ "language_model.layers.1.self_attn.v_proj",
69
+ "35.self_attn.k_proj",
70
+ "language_model.layers.17.self_attn.v_proj",
71
+ "language_model.layers.8.self_attn.k_proj",
72
+ "language_model.layers.21.self_attn.q_proj",
73
+ "39.self_attn.k_proj",
74
+ "language_model.layers.12.self_attn.k_proj",
75
+ "language_model.layers.14.self_attn.q_proj",
76
+ "language_model.layers.11.self_attn.v_proj",
77
+ "34.self_attn.k_proj",
78
+ "language_model.layers.10.self_attn.k_proj",
79
+ "language_model.layers.5.self_attn.k_proj",
80
+ "language_model.layers.17.self_attn.q_proj",
81
+ "44.self_attn.k_proj",
82
+ "language_model.layers.20.self_attn.v_proj",
83
+ "30.self_attn.v_proj",
84
+ "29.self_attn.v_proj",
85
+ "language_model.layers.4.self_attn.q_proj",
86
+ "language_model.layers.9.self_attn.q_proj",
87
+ "language_model.layers.4.self_attn.v_proj",
88
+ "27.self_attn.k_proj",
89
+ "38.self_attn.v_proj",
90
+ "language_model.layers.18.self_attn.k_proj",
91
+ "down_proj",
92
+ "45.self_attn.v_proj",
93
+ "language_model.layers.26.self_attn.q_proj",
94
+ "language_model.layers.2.self_attn.q_proj",
95
+ "language_model.layers.3.self_attn.q_proj",
96
+ "language_model.layers.2.self_attn.v_proj",
97
+ "language_model.layers.16.self_attn.q_proj",
98
+ "language_model.layers.2.self_attn.k_proj",
99
+ "27.self_attn.v_proj",
100
+ "33.self_attn.q_proj",
101
+ "43.self_attn.q_proj",
102
+ "31.self_attn.v_proj",
103
+ "33.self_attn.v_proj",
104
+ "42.self_attn.q_proj",
105
+ "28.self_attn.k_proj",
106
+ "language_model.layers.13.self_attn.v_proj",
107
+ "46.self_attn.v_proj",
108
+ "language_model.layers.19.self_attn.k_proj",
109
+ "language_model.layers.0.self_attn.q_proj",
110
+ "language_model.layers.3.self_attn.k_proj",
111
+ "language_model.layers.14.self_attn.v_proj",
112
+ "37.self_attn.v_proj",
113
+ "43.self_attn.v_proj",
114
+ "39.self_attn.q_proj",
115
+ "language_model.layers.5.self_attn.q_proj",
116
+ "language_model.layers.24.self_attn.q_proj",
117
+ "language_model.layers.24.self_attn.k_proj",
118
+ "47.self_attn.q_proj",
119
+ "30.self_attn.q_proj",
120
+ "language_model.layers.4.self_attn.k_proj",
121
+ "33.self_attn.k_proj",
122
+ "40.self_attn.k_proj",
123
+ "language_model.layers.10.self_attn.v_proj",
124
+ "language_model.layers.25.self_attn.v_proj",
125
+ "37.self_attn.k_proj",
126
+ "language_model.layers.19.self_attn.v_proj",
127
+ "41.self_attn.q_proj",
128
+ "language_model.layers.22.self_attn.k_proj",
129
+ "46.self_attn.q_proj",
130
+ "28.self_attn.q_proj",
131
+ "language_model.layers.7.self_attn.k_proj",
132
+ "language_model.layers.21.self_attn.k_proj",
133
+ "language_model.layers.6.self_attn.k_proj",
134
+ "28.self_attn.v_proj",
135
+ "language_model.layers.3.self_attn.v_proj",
136
+ "36.self_attn.q_proj",
137
+ "language_model.layers.23.self_attn.k_proj",
138
+ "language_model.layers.0.self_attn.v_proj",
139
+ "language_model.layers.15.self_attn.v_proj",
140
+ "language_model.layers.15.self_attn.q_proj",
141
+ "gate_proj",
142
+ "34.self_attn.q_proj",
143
+ "language_model.layers.18.self_attn.q_proj",
144
+ "43.self_attn.k_proj",
145
+ "language_model.layers.11.self_attn.k_proj",
146
+ "language_model.layers.22.self_attn.q_proj",
147
+ "language_model.layers.17.self_attn.k_proj",
148
+ "36.self_attn.v_proj",
149
+ "language_model.layers.26.self_attn.k_proj",
150
+ "45.self_attn.k_proj",
151
+ "language_model.layers.12.self_attn.q_proj",
152
+ "37.self_attn.q_proj",
153
+ "44.self_attn.q_proj",
154
+ "language_model.layers.14.self_attn.k_proj",
155
+ "language_model.layers.23.self_attn.v_proj",
156
+ "42.self_attn.v_proj",
157
+ "47.self_attn.k_proj",
158
+ "language_model.layers.20.self_attn.q_proj",
159
+ "language_model.layers.5.self_attn.v_proj",
160
+ "language_model.layers.20.self_attn.k_proj",
161
+ "language_model.layers.24.self_attn.v_proj",
162
+ "32.self_attn.q_proj",
163
+ "language_model.layers.13.self_attn.q_proj",
164
+ "30.self_attn.k_proj",
165
+ "39.self_attn.v_proj",
166
+ "32.self_attn.k_proj",
167
+ "language_model.layers.1.self_attn.q_proj",
168
+ "language_model.layers.0.self_attn.k_proj",
169
+ "language_model.layers.19.self_attn.q_proj",
170
+ "27.self_attn.q_proj",
171
+ "language_model.layers.8.self_attn.q_proj",
172
+ "language_model.layers.25.self_attn.k_proj",
173
+ "language_model.layers.8.self_attn.v_proj",
174
+ "language_model.layers.16.self_attn.k_proj",
175
+ "38.self_attn.k_proj",
176
+ "language_model.layers.16.self_attn.v_proj",
177
+ "29.self_attn.k_proj",
178
+ "language_model.layers.13.self_attn.k_proj",
179
+ "41.self_attn.v_proj",
180
+ "language_model.layers.23.self_attn.q_proj",
181
+ "42.self_attn.k_proj"
182
+ ],
183
+ "target_parameters": null,
184
+ "task_type": "CAUSAL_LM",
185
+ "trainable_token_indices": null,
186
+ "use_bdlora": null,
187
+ "use_dora": false,
188
+ "use_qalora": false,
189
+ "use_rslora": false,
190
+ "velora_config": null
191
+ }
rl_grpo/charter_real_4x_thinking/checkpoint-64/adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6baeaae9a85d02a324d39fe43703e4ef1cab6d535d6c2eff6b799a68f1d04422
3
+ size 523864568
rl_grpo/charter_real_4x_thinking/checkpoint.json ADDED
@@ -0,0 +1,132 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "experiment": "scimt-train:dispatch-rl-charter_real_4x_thinking",
3
+ "spec": null,
4
+ "kind": null,
5
+ "note": "Spec-free training stage (scimt.train.train_dataset) \u2014 a post-training link in a staged chain, not a spec install.",
6
+ "train": {
7
+ "data": "/workspace/rl3t_a/data/thinking/train.jsonl",
8
+ "dataset_meta": {
9
+ "adhoc": true
10
+ },
11
+ "stage": null,
12
+ "seed": 42,
13
+ "load_checkpoint_path": "/workspace/rl3t_a/parent",
14
+ "grpo": {
15
+ "episodes": 8192,
16
+ "group_size": 8,
17
+ "max_prompt_length": 3072,
18
+ "max_completion_length": 1024,
19
+ "per_device_batch_size": 2,
20
+ "gradient_accumulation_steps": 16,
21
+ "steps_per_generation": 16,
22
+ "learning_rate": 1e-05,
23
+ "temperature": 0.7,
24
+ "loss_type": "dr_grpo",
25
+ "beta": 0.0,
26
+ "vllm": "colocate",
27
+ "vllm_gpu_memory_utilization": 0.38,
28
+ "vllm_max_model_len": 4096,
29
+ "mask_truncated_completions": true,
30
+ "log_completions": true,
31
+ "num_completions_to_print": 2,
32
+ "log_unique_prompts": true,
33
+ "checkpoint_fractions": [
34
+ 0.0625,
35
+ 0.125,
36
+ 0.25,
37
+ 0.5,
38
+ 1.0
39
+ ],
40
+ "report_to": [],
41
+ "reward_func": "experiments.prior_coins.dispatch_rl_reward_v2:reward_thinking",
42
+ "resume_from_checkpoint": null,
43
+ "rollout_log_dir": "/workspace/rl3t_a/training/charter_real_4x_thinking/logs",
44
+ "abort_log_path": null,
45
+ "validation_dataset_path": null,
46
+ "abort_eval_func": null,
47
+ "parent_agreement": null,
48
+ "parent_reward": null,
49
+ "parent_completion_length": null,
50
+ "zero_std_warmup_fraction": 0.1,
51
+ "completion_length_window": 1024
52
+ },
53
+ "lora": {
54
+ "r": 32,
55
+ "alpha": 64,
56
+ "dropout": 0.0,
57
+ "target_linear": true,
58
+ "target_modules": null
59
+ }
60
+ },
61
+ "run_name": "dispatch-rl-charter_real_4x_thinking",
62
+ "pointer_file": "/workspace/rl3t_a/training/charter_real_4x_thinking/train/ckpt_dispatch-rl-charter_real_4x_thinking.txt",
63
+ "backend": "hf_grpo",
64
+ "sampler": "/workspace/rl3t_a/training/charter_real_4x_thinking/train/sampler",
65
+ "state": "/workspace/rl3t_a/training/charter_real_4x_thinking/train/trainer/checkpoint-256",
66
+ "model": "google/gemma-3-12b-pt",
67
+ "meta": {
68
+ "experiment": "scimt-train:dispatch-rl-charter_real_4x_thinking",
69
+ "spec": null,
70
+ "kind": null,
71
+ "note": "Spec-free training stage (scimt.train.train_dataset) \u2014 a post-training link in a staged chain, not a spec install.",
72
+ "train": {
73
+ "data": "/workspace/rl3t_a/data/thinking/train.jsonl",
74
+ "dataset_meta": {
75
+ "adhoc": true
76
+ },
77
+ "stage": null,
78
+ "seed": 42,
79
+ "load_checkpoint_path": "/workspace/rl3t_a/parent",
80
+ "grpo": {
81
+ "episodes": 8192,
82
+ "group_size": 8,
83
+ "max_prompt_length": 3072,
84
+ "max_completion_length": 1024,
85
+ "per_device_batch_size": 2,
86
+ "gradient_accumulation_steps": 16,
87
+ "steps_per_generation": 16,
88
+ "learning_rate": 1e-05,
89
+ "temperature": 0.7,
90
+ "loss_type": "dr_grpo",
91
+ "beta": 0.0,
92
+ "vllm": "colocate",
93
+ "vllm_gpu_memory_utilization": 0.38,
94
+ "vllm_max_model_len": 4096,
95
+ "mask_truncated_completions": true,
96
+ "log_completions": true,
97
+ "num_completions_to_print": 2,
98
+ "log_unique_prompts": true,
99
+ "checkpoint_fractions": [
100
+ 0.0625,
101
+ 0.125,
102
+ 0.25,
103
+ 0.5,
104
+ 1.0
105
+ ],
106
+ "report_to": [],
107
+ "reward_func": "experiments.prior_coins.dispatch_rl_reward_v2:reward_thinking",
108
+ "resume_from_checkpoint": null,
109
+ "rollout_log_dir": "/workspace/rl3t_a/training/charter_real_4x_thinking/logs",
110
+ "abort_log_path": null,
111
+ "validation_dataset_path": null,
112
+ "abort_eval_func": null,
113
+ "parent_agreement": null,
114
+ "parent_reward": null,
115
+ "parent_completion_length": null,
116
+ "zero_std_warmup_fraction": 0.1,
117
+ "completion_length_window": 1024
118
+ },
119
+ "lora": {
120
+ "r": 32,
121
+ "alpha": 64,
122
+ "dropout": 0.0,
123
+ "target_linear": true,
124
+ "target_modules": null
125
+ }
126
+ },
127
+ "run_name": "dispatch-rl-charter_real_4x_thinking",
128
+ "pointer_file": "/workspace/rl3t_a/training/charter_real_4x_thinking/train/ckpt_dispatch-rl-charter_real_4x_thinking.txt"
129
+ },
130
+ "sampler_path": "/workspace/rl3t_a/training/charter_real_4x_thinking/train/sampler",
131
+ "state_path": "/workspace/rl3t_a/training/charter_real_4x_thinking/train/trainer/checkpoint-256"
132
+ }
rl_grpo/charter_real_4x_thinking/logs/raw_rollouts.rank-0.jsonl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8e16d2971f05044b588d9710e7c85e68e46677a745caa9dd5ae6394e6587a505
3
+ size 208767168