TheBlackCat22 commited on
Commit
80fe913
·
verified ·
1 Parent(s): bf6715e

Delete 20260822_095129

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. 20260822_095129/global_step_250/actor/added_tokens.json +0 -28
  2. 20260822_095129/global_step_250/actor/config.json +0 -31
  3. 20260822_095129/global_step_250/actor/extra_state_world_size_8_rank_0.pt +0 -3
  4. 20260822_095129/global_step_250/actor/extra_state_world_size_8_rank_1.pt +0 -3
  5. 20260822_095129/global_step_250/actor/extra_state_world_size_8_rank_2.pt +0 -3
  6. 20260822_095129/global_step_250/actor/extra_state_world_size_8_rank_3.pt +0 -3
  7. 20260822_095129/global_step_250/actor/extra_state_world_size_8_rank_4.pt +0 -3
  8. 20260822_095129/global_step_250/actor/extra_state_world_size_8_rank_5.pt +0 -3
  9. 20260822_095129/global_step_250/actor/extra_state_world_size_8_rank_6.pt +0 -3
  10. 20260822_095129/global_step_250/actor/extra_state_world_size_8_rank_7.pt +0 -3
  11. 20260822_095129/global_step_250/actor/generation_config.json +0 -13
  12. 20260822_095129/global_step_250/actor/merges.txt +0 -0
  13. 20260822_095129/global_step_250/actor/special_tokens_map.json +0 -31
  14. 20260822_095129/global_step_250/actor/tokenizer.json +0 -3
  15. 20260822_095129/global_step_250/actor/tokenizer_config.json +0 -240
  16. 20260822_095129/global_step_250/actor/vocab.json +0 -0
  17. 20260822_095129/global_step_250/data.pt +0 -3
  18. 20260822_095129/latest_checkpointed_iteration.txt +0 -1
  19. 20260822_095129/outputs/.hydra/config.yaml +0 -334
  20. 20260822_095129/outputs/.hydra/hydra.yaml +0 -232
  21. 20260822_095129/outputs/.hydra/overrides.yaml +0 -76
  22. 20260822_095129/outputs/main_ppo.log +0 -0
  23. 20260822_095129/rollout/1.jsonl +0 -3
  24. 20260822_095129/rollout/10.jsonl +0 -3
  25. 20260822_095129/rollout/101.jsonl +0 -3
  26. 20260822_095129/rollout/103.jsonl +0 -3
  27. 20260822_095129/rollout/106.jsonl +0 -3
  28. 20260822_095129/rollout/107.jsonl +0 -3
  29. 20260822_095129/rollout/108.jsonl +0 -3
  30. 20260822_095129/rollout/109.jsonl +0 -3
  31. 20260822_095129/rollout/110.jsonl +0 -3
  32. 20260822_095129/rollout/112.jsonl +0 -3
  33. 20260822_095129/rollout/113.jsonl +0 -3
  34. 20260822_095129/rollout/115.jsonl +0 -3
  35. 20260822_095129/rollout/116.jsonl +0 -3
  36. 20260822_095129/rollout/118.jsonl +0 -3
  37. 20260822_095129/rollout/12.jsonl +0 -3
  38. 20260822_095129/rollout/121.jsonl +0 -3
  39. 20260822_095129/rollout/122.jsonl +0 -3
  40. 20260822_095129/rollout/123.jsonl +0 -3
  41. 20260822_095129/rollout/124.jsonl +0 -3
  42. 20260822_095129/rollout/129.jsonl +0 -3
  43. 20260822_095129/rollout/13.jsonl +0 -3
  44. 20260822_095129/rollout/130.jsonl +0 -3
  45. 20260822_095129/rollout/131.jsonl +0 -3
  46. 20260822_095129/rollout/132.jsonl +0 -3
  47. 20260822_095129/rollout/133.jsonl +0 -3
  48. 20260822_095129/rollout/134.jsonl +0 -3
  49. 20260822_095129/rollout/135.jsonl +0 -3
  50. 20260822_095129/rollout/136.jsonl +0 -3
20260822_095129/global_step_250/actor/added_tokens.json DELETED
@@ -1,28 +0,0 @@
1
- {
2
- "</think>": 151668,
3
- "</tool_call>": 151658,
4
- "</tool_response>": 151666,
5
- "<think>": 151667,
6
- "<tool_call>": 151657,
7
- "<tool_response>": 151665,
8
- "<|box_end|>": 151649,
9
- "<|box_start|>": 151648,
10
- "<|endoftext|>": 151643,
11
- "<|file_sep|>": 151664,
12
- "<|fim_middle|>": 151660,
13
- "<|fim_pad|>": 151662,
14
- "<|fim_prefix|>": 151659,
15
- "<|fim_suffix|>": 151661,
16
- "<|im_end|>": 151645,
17
- "<|im_start|>": 151644,
18
- "<|image_pad|>": 151655,
19
- "<|object_ref_end|>": 151647,
20
- "<|object_ref_start|>": 151646,
21
- "<|quad_end|>": 151651,
22
- "<|quad_start|>": 151650,
23
- "<|repo_name|>": 151663,
24
- "<|video_pad|>": 151656,
25
- "<|vision_end|>": 151653,
26
- "<|vision_pad|>": 151654,
27
- "<|vision_start|>": 151652
28
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
20260822_095129/global_step_250/actor/config.json DELETED
@@ -1,31 +0,0 @@
1
- {
2
- "_attn_implementation_autoset": true,
3
- "architectures": [
4
- "Qwen3ForCausalLM"
5
- ],
6
- "attention_bias": false,
7
- "attention_dropout": 0.0,
8
- "eos_token_id": 151645,
9
- "head_dim": 128,
10
- "hidden_act": "silu",
11
- "hidden_size": 2560,
12
- "initializer_range": 0.02,
13
- "intermediate_size": 9728,
14
- "max_position_embeddings": 40960,
15
- "max_window_layers": 36,
16
- "model_type": "qwen3",
17
- "num_attention_heads": 32,
18
- "num_hidden_layers": 36,
19
- "num_key_value_heads": 8,
20
- "pad_token_id": 151643,
21
- "rms_norm_eps": 1e-06,
22
- "rope_scaling": null,
23
- "rope_theta": 1000000,
24
- "sliding_window": null,
25
- "tie_word_embeddings": true,
26
- "torch_dtype": "float32",
27
- "transformers_version": "4.51.3",
28
- "use_cache": true,
29
- "use_sliding_window": false,
30
- "vocab_size": 151936
31
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
20260822_095129/global_step_250/actor/extra_state_world_size_8_rank_0.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:125313d2ba1ae3026fed98348454ed9a80395cf0878bb7e89d5b0dadddd3eab2
3
- size 14632
 
 
 
 
20260822_095129/global_step_250/actor/extra_state_world_size_8_rank_1.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:cfb1def61dad117fa0f3d9f855f085aae04b7b4f6d9764020d70fd1919ccf8d2
3
- size 14632
 
 
 
 
20260822_095129/global_step_250/actor/extra_state_world_size_8_rank_2.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:2f06b73d6b1af10fa2f5deef18998b4dbc1d1fc85b3dec1007bc62a0a7e22e81
3
- size 14632
 
 
 
 
20260822_095129/global_step_250/actor/extra_state_world_size_8_rank_3.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:b4324a60cd185bb7cb4374a071b2f4571d84dfa741cd5d8dfeabfb64f812cf1b
3
- size 14632
 
 
 
 
20260822_095129/global_step_250/actor/extra_state_world_size_8_rank_4.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:a89c042ce6dacdc0b71194975f1a873095109fe8a773413898e82ef50b9b22e7
3
- size 14632
 
 
 
 
20260822_095129/global_step_250/actor/extra_state_world_size_8_rank_5.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:ecccf6999940cc36315cd7600cccaae3d5c6d4d793186763a14d3cd8607f5511
3
- size 14632
 
 
 
 
20260822_095129/global_step_250/actor/extra_state_world_size_8_rank_6.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:e190d4c56d2ef9b09931ecc5fe8bf15d5b78bb55ba6faeae7cc9c2c25a159f26
3
- size 14632
 
 
 
 
20260822_095129/global_step_250/actor/extra_state_world_size_8_rank_7.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:c3ac6e2b2ee34668b4736bc5a13fa122921db4f156faa8ebc9caacc9a674ea41
3
- size 14632
 
 
 
 
20260822_095129/global_step_250/actor/generation_config.json DELETED
@@ -1,13 +0,0 @@
1
- {
2
- "bos_token_id": 151643,
3
- "do_sample": true,
4
- "eos_token_id": [
5
- 151645,
6
- 151643
7
- ],
8
- "pad_token_id": 151643,
9
- "temperature": 0.6,
10
- "top_k": 20,
11
- "top_p": 0.95,
12
- "transformers_version": "4.51.3"
13
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
20260822_095129/global_step_250/actor/merges.txt DELETED
The diff for this file is too large to render. See raw diff
 
20260822_095129/global_step_250/actor/special_tokens_map.json DELETED
@@ -1,31 +0,0 @@
1
- {
2
- "additional_special_tokens": [
3
- "<|im_start|>",
4
- "<|im_end|>",
5
- "<|object_ref_start|>",
6
- "<|object_ref_end|>",
7
- "<|box_start|>",
8
- "<|box_end|>",
9
- "<|quad_start|>",
10
- "<|quad_end|>",
11
- "<|vision_start|>",
12
- "<|vision_end|>",
13
- "<|vision_pad|>",
14
- "<|image_pad|>",
15
- "<|video_pad|>"
16
- ],
17
- "eos_token": {
18
- "content": "<|im_end|>",
19
- "lstrip": false,
20
- "normalized": false,
21
- "rstrip": false,
22
- "single_word": false
23
- },
24
- "pad_token": {
25
- "content": "<|endoftext|>",
26
- "lstrip": false,
27
- "normalized": false,
28
- "rstrip": false,
29
- "single_word": false
30
- }
31
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
20260822_095129/global_step_250/actor/tokenizer.json DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4
3
- size 11422654
 
 
 
 
20260822_095129/global_step_250/actor/tokenizer_config.json DELETED
@@ -1,240 +0,0 @@
1
- {
2
- "add_bos_token": false,
3
- "add_prefix_space": false,
4
- "added_tokens_decoder": {
5
- "151643": {
6
- "content": "<|endoftext|>",
7
- "lstrip": false,
8
- "normalized": false,
9
- "rstrip": false,
10
- "single_word": false,
11
- "special": true
12
- },
13
- "151644": {
14
- "content": "<|im_start|>",
15
- "lstrip": false,
16
- "normalized": false,
17
- "rstrip": false,
18
- "single_word": false,
19
- "special": true
20
- },
21
- "151645": {
22
- "content": "<|im_end|>",
23
- "lstrip": false,
24
- "normalized": false,
25
- "rstrip": false,
26
- "single_word": false,
27
- "special": true
28
- },
29
- "151646": {
30
- "content": "<|object_ref_start|>",
31
- "lstrip": false,
32
- "normalized": false,
33
- "rstrip": false,
34
- "single_word": false,
35
- "special": true
36
- },
37
- "151647": {
38
- "content": "<|object_ref_end|>",
39
- "lstrip": false,
40
- "normalized": false,
41
- "rstrip": false,
42
- "single_word": false,
43
- "special": true
44
- },
45
- "151648": {
46
- "content": "<|box_start|>",
47
- "lstrip": false,
48
- "normalized": false,
49
- "rstrip": false,
50
- "single_word": false,
51
- "special": true
52
- },
53
- "151649": {
54
- "content": "<|box_end|>",
55
- "lstrip": false,
56
- "normalized": false,
57
- "rstrip": false,
58
- "single_word": false,
59
- "special": true
60
- },
61
- "151650": {
62
- "content": "<|quad_start|>",
63
- "lstrip": false,
64
- "normalized": false,
65
- "rstrip": false,
66
- "single_word": false,
67
- "special": true
68
- },
69
- "151651": {
70
- "content": "<|quad_end|>",
71
- "lstrip": false,
72
- "normalized": false,
73
- "rstrip": false,
74
- "single_word": false,
75
- "special": true
76
- },
77
- "151652": {
78
- "content": "<|vision_start|>",
79
- "lstrip": false,
80
- "normalized": false,
81
- "rstrip": false,
82
- "single_word": false,
83
- "special": true
84
- },
85
- "151653": {
86
- "content": "<|vision_end|>",
87
- "lstrip": false,
88
- "normalized": false,
89
- "rstrip": false,
90
- "single_word": false,
91
- "special": true
92
- },
93
- "151654": {
94
- "content": "<|vision_pad|>",
95
- "lstrip": false,
96
- "normalized": false,
97
- "rstrip": false,
98
- "single_word": false,
99
- "special": true
100
- },
101
- "151655": {
102
- "content": "<|image_pad|>",
103
- "lstrip": false,
104
- "normalized": false,
105
- "rstrip": false,
106
- "single_word": false,
107
- "special": true
108
- },
109
- "151656": {
110
- "content": "<|video_pad|>",
111
- "lstrip": false,
112
- "normalized": false,
113
- "rstrip": false,
114
- "single_word": false,
115
- "special": true
116
- },
117
- "151657": {
118
- "content": "<tool_call>",
119
- "lstrip": false,
120
- "normalized": false,
121
- "rstrip": false,
122
- "single_word": false,
123
- "special": false
124
- },
125
- "151658": {
126
- "content": "</tool_call>",
127
- "lstrip": false,
128
- "normalized": false,
129
- "rstrip": false,
130
- "single_word": false,
131
- "special": false
132
- },
133
- "151659": {
134
- "content": "<|fim_prefix|>",
135
- "lstrip": false,
136
- "normalized": false,
137
- "rstrip": false,
138
- "single_word": false,
139
- "special": false
140
- },
141
- "151660": {
142
- "content": "<|fim_middle|>",
143
- "lstrip": false,
144
- "normalized": false,
145
- "rstrip": false,
146
- "single_word": false,
147
- "special": false
148
- },
149
- "151661": {
150
- "content": "<|fim_suffix|>",
151
- "lstrip": false,
152
- "normalized": false,
153
- "rstrip": false,
154
- "single_word": false,
155
- "special": false
156
- },
157
- "151662": {
158
- "content": "<|fim_pad|>",
159
- "lstrip": false,
160
- "normalized": false,
161
- "rstrip": false,
162
- "single_word": false,
163
- "special": false
164
- },
165
- "151663": {
166
- "content": "<|repo_name|>",
167
- "lstrip": false,
168
- "normalized": false,
169
- "rstrip": false,
170
- "single_word": false,
171
- "special": false
172
- },
173
- "151664": {
174
- "content": "<|file_sep|>",
175
- "lstrip": false,
176
- "normalized": false,
177
- "rstrip": false,
178
- "single_word": false,
179
- "special": false
180
- },
181
- "151665": {
182
- "content": "<tool_response>",
183
- "lstrip": false,
184
- "normalized": false,
185
- "rstrip": false,
186
- "single_word": false,
187
- "special": false
188
- },
189
- "151666": {
190
- "content": "</tool_response>",
191
- "lstrip": false,
192
- "normalized": false,
193
- "rstrip": false,
194
- "single_word": false,
195
- "special": false
196
- },
197
- "151667": {
198
- "content": "<think>",
199
- "lstrip": false,
200
- "normalized": false,
201
- "rstrip": false,
202
- "single_word": false,
203
- "special": false
204
- },
205
- "151668": {
206
- "content": "</think>",
207
- "lstrip": false,
208
- "normalized": false,
209
- "rstrip": false,
210
- "single_word": false,
211
- "special": false
212
- }
213
- },
214
- "additional_special_tokens": [
215
- "<|im_start|>",
216
- "<|im_end|>",
217
- "<|object_ref_start|>",
218
- "<|object_ref_end|>",
219
- "<|box_start|>",
220
- "<|box_end|>",
221
- "<|quad_start|>",
222
- "<|quad_end|>",
223
- "<|vision_start|>",
224
- "<|vision_end|>",
225
- "<|vision_pad|>",
226
- "<|image_pad|>",
227
- "<|video_pad|>"
228
- ],
229
- "bos_token": null,
230
- "chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n{%- endfor %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is string %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- else %}\n {%- if '</think>' in content %}\n {%- set reasoning_content = content.split('</think>')[0].rstrip('\\n').split('<think>')[-1].lstrip('\\n') %}\n {%- set content = content.split('</think>')[-1].lstrip('\\n') %}\n {%- endif %}\n {%- endif %}\n {%- if loop.index0 > ns.last_query_index %}\n {%- if loop.last or (not loop.last and reasoning_content) %}\n {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content.strip('\\n') + '\\n</think>\\n\\n' + content.lstrip('\\n') }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- content }}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n {%- if enable_thinking is defined and enable_thinking is false %}\n {{- '<think>\\n\\n</think>\\n\\n' }}\n {%- endif %}\n{%- endif %}",
231
- "clean_up_tokenization_spaces": false,
232
- "eos_token": "<|im_end|>",
233
- "errors": "replace",
234
- "extra_special_tokens": {},
235
- "model_max_length": 131072,
236
- "pad_token": "<|endoftext|>",
237
- "split_special_tokens": false,
238
- "tokenizer_class": "Qwen2Tokenizer",
239
- "unk_token": null
240
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
20260822_095129/global_step_250/actor/vocab.json DELETED
The diff for this file is too large to render. See raw diff
 
20260822_095129/global_step_250/data.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:5f99f27d377ba0e1bfc18314efb82d03e95eca73932b51bd1325227e76cba7c1
3
- size 1492
 
 
 
 
20260822_095129/latest_checkpointed_iteration.txt DELETED
@@ -1 +0,0 @@
1
- 250
 
 
20260822_095129/outputs/.hydra/config.yaml DELETED
@@ -1,334 +0,0 @@
1
- data:
2
- tokenizer: null
3
- use_shm: false
4
- train_files: /scratch/user/sushil22_tamu.edu/projects/TreeHCA/rl_datasets/hotpotqa/train.parquet
5
- val_files:
6
- - /scratch/user/sushil22_tamu.edu/projects/TreeHCA/rl_datasets/hotpotqa/test_512.parquet
7
- prompt_key: prompt
8
- reward_fn_key: data_source
9
- max_prompt_length: 2000
10
- max_response_length: 6192
11
- train_batch_size: 64
12
- val_batch_size: null
13
- return_raw_input_ids: false
14
- return_raw_chat: false
15
- return_full_prompt: false
16
- shuffle: true
17
- seed: 42
18
- filter_overlong_prompts: false
19
- filter_overlong_prompts_workers: 1
20
- truncation: error
21
- image_key: images
22
- video_key: videos
23
- trust_remote_code: false
24
- custom_cls:
25
- path: null
26
- name: null
27
- algorithm:
28
- gamma: 1.0
29
- lam: 1.0
30
- adv_estimator: grpo
31
- norm_adv_by_std_in_grpo: true
32
- use_kl_in_reward: false
33
- kl_penalty: kl
34
- kl_ctrl:
35
- type: fixed
36
- kl_coef: 0.0
37
- horizon: 10000
38
- target_kl: 0.1
39
- use_pf_ppo: false
40
- pf_ppo:
41
- reweight_method: pow
42
- weight_pow: 2.0
43
- actor_rollout_ref:
44
- hybrid_engine: true
45
- model:
46
- path: /scratch/user/sushil22_tamu.edu/projects/TreeHCA/models/Qwen3-4B
47
- use_shm: false
48
- external_lib: null
49
- override_config: {}
50
- enable_gradient_checkpointing: true
51
- enable_activation_offload: false
52
- use_remove_padding: true
53
- lora_rank: 0
54
- lora_alpha: 16
55
- target_modules: all-linear
56
- use_liger: false
57
- use_fused_kernels: false
58
- trust_remote_code: false
59
- actor:
60
- policy_loss: grpo
61
- enable_entropy_balanced_clipping: false
62
- enable_entropy_balanced_advantage: false
63
- strategy: fsdp
64
- ppo_mini_batch_size: 8
65
- ppo_micro_batch_size: null
66
- ppo_micro_batch_size_per_gpu: null
67
- use_dynamic_bsz: true
68
- ppo_max_token_len_per_gpu: 16384
69
- grad_clip: 1.0
70
- clip_ratio: 0.2
71
- clip_ratio_low: 0.2
72
- clip_ratio_high: 0.2
73
- clip_ratio_c: 3.0
74
- loss_agg_mode: token-mean
75
- entropy_coeff: 0
76
- use_kl_loss: true
77
- use_torch_compile: true
78
- kl_loss_coef: 0.0
79
- kl_loss_type: low_var_kl
80
- ppo_epochs: 1
81
- shuffle: false
82
- ulysses_sequence_parallel_size: 1
83
- checkpoint:
84
- contents:
85
- - model
86
- - optimizer
87
- - extra
88
- save_contents:
89
- - hf_model
90
- - extra
91
- optim:
92
- lr: 1.0e-06
93
- lr_warmup_steps: -1
94
- lr_warmup_steps_ratio: 0.0
95
- min_lr_ratio: 0.0
96
- num_cycles: 0.5
97
- warmup_style: constant
98
- total_training_steps: -1
99
- weight_decay: 0.01
100
- fsdp_config:
101
- wrap_policy:
102
- min_num_params: 0
103
- param_offload: false
104
- optimizer_offload: false
105
- offload_policy: false
106
- reshard_after_forward: true
107
- fsdp_size: -1
108
- ref:
109
- strategy: fsdp
110
- fsdp_config:
111
- param_offload: true
112
- reshard_after_forward: true
113
- wrap_policy:
114
- min_num_params: 0
115
- use_torch_compile: ${actor_rollout_ref.actor.use_torch_compile}
116
- log_prob_micro_batch_size: null
117
- log_prob_micro_batch_size_per_gpu: null
118
- log_prob_use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz}
119
- log_prob_max_token_len_per_gpu: 32768
120
- ulysses_sequence_parallel_size: ${actor_rollout_ref.actor.ulysses_sequence_parallel_size}
121
- rollout:
122
- name: vllm
123
- mode: sync_with_tool_tree_treegrpo
124
- chat_scheduler: null
125
- temperature: 1.0
126
- top_k: -1
127
- top_p: 1
128
- use_fire_sampling: false
129
- prompt_length: ${data.max_prompt_length}
130
- response_length: ${data.max_response_length}
131
- initial_rollouts: 1
132
- expansion_mode: random
133
- expansion_iterations: 1
134
- beam_size: 10
135
- samples_per_tree: 11
136
- counterfactual_log_drop_smooth: 0.05
137
- counterfactual_log_drop_floor: 0.14
138
- data: ${data}
139
- use_kl_in_reward: ${algorithm.use_kl_in_reward}
140
- kl_penalty: ${algorithm.kl_penalty}
141
- adv_estimator: ${algorithm.adv_estimator}
142
- gamma: ${algorithm.gamma}
143
- lam: ${algorithm.lam}
144
- norm_adv_by_std_in_grpo: ${algorithm.norm_adv_by_std_in_grpo}
145
- kl_ctrl: ${algorithm.kl_ctrl}
146
- reward_model: ${reward_model}
147
- custom_reward_function: ${custom_reward_function}
148
- leaf_value_norm: false
149
- node_value_mode: leaf_mean
150
- node_adv_mode: diff_global
151
- hindsight_gt_clamp: 0.01
152
- hindsight_self_normalize: true
153
- leaf_value_source: raw
154
- leaf_reward_metric: f1
155
- score_leaves_on_finish: false
156
- prune_incorrect_first: false
157
- rescue_expansion_threshold: 0.0
158
- format_error_penalty: 0.0
159
- entropy_mixing_method: no_entropy
160
- dtype: bfloat16
161
- gpu_memory_utilization: 0.6
162
- ignore_eos: false
163
- enforce_eager: true
164
- free_cache_engine: true
165
- load_format: dummy_dtensor
166
- layered_summon: false
167
- tensor_model_parallel_size: 1
168
- max_num_batched_tokens: 25000
169
- max_model_len: null
170
- max_num_seqs: 1024
171
- log_prob_micro_batch_size: null
172
- log_prob_micro_batch_size_per_gpu: null
173
- log_prob_use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz}
174
- log_prob_max_token_len_per_gpu: 32768
175
- disable_log_stats: true
176
- enable_chunked_prefill: true
177
- do_sample: true
178
- 'n': 22
179
- branch_probability: 0.5
180
- entropy_weight: 0.2
181
- engine_kwargs:
182
- vllm:
183
- swap_space: null
184
- sglang:
185
- attention_backend: null
186
- val_kwargs:
187
- top_k: -1
188
- top_p: 1.0
189
- temperature: 0
190
- 'n': 1
191
- do_sample: false
192
- multi_turn:
193
- enable: true
194
- max_turns: null
195
- tool_config_path: /scratch/user/sushil22_tamu.edu/projects/TreeHCA/TreeHCA/verl_treehca/examples/sglang_multiturn/config/tool_config/search_tool_config.yaml
196
- format: chatml
197
- enable_dynamic_rollouts: false
198
- tools:
199
- call_limit: 6
200
- max_workers: 32
201
- timeout: 120
202
- retry_count: 4
203
- verbose_logging: true
204
- fail_on_error: true
205
- tool_instances:
206
- python:
207
- class_path: verl.workers.rollout.tools.python_tool_turn.PythonTool
208
- params:
209
- conda_path: /scratch/user/sushil22_tamu.edu/miniconda3
210
- conda_env: treehca_env
211
- search:
212
- class_path: verl.workers.rollout.tools.search_tool.BingSearchTool
213
- params:
214
- api_key: unused_local_rag_server
215
- zone: serp_api_test
216
- max_results: 10
217
- result_length: 1000
218
- location: cn
219
- cache_file: null
220
- async_cache_write: true
221
- reduce_gt_prob_context: false
222
- num_trees_per_prompt: 2
223
- reward_mode: base
224
- compute_gt_prob_stats: false
225
- critic:
226
- rollout_n: ${actor_rollout_ref.rollout.n}
227
- strategy: fsdp
228
- optim:
229
- lr: 1.0e-05
230
- lr_warmup_steps_ratio: 0.0
231
- min_lr_ratio: null
232
- warmup_style: constant
233
- total_training_steps: -1
234
- weight_decay: 0.01
235
- model:
236
- path: ~/models/deepseek-llm-7b-chat
237
- use_shm: false
238
- tokenizer_path: ${actor_rollout_ref.model.path}
239
- override_config: {}
240
- external_lib: ${actor_rollout_ref.model.external_lib}
241
- enable_gradient_checkpointing: true
242
- enable_activation_offload: false
243
- use_remove_padding: false
244
- trust_remote_code: ${actor_rollout_ref.model.trust_remote_code}
245
- fsdp_config:
246
- param_offload: false
247
- optimizer_offload: false
248
- offload_policy: false
249
- reshard_after_forward: true
250
- wrap_policy:
251
- min_num_params: 0
252
- fsdp_size: -1
253
- lora_rank: 0
254
- lora_alpha: 16
255
- target_modules: all-linear
256
- ppo_mini_batch_size: ${actor_rollout_ref.actor.ppo_mini_batch_size}
257
- ppo_micro_batch_size: null
258
- ppo_micro_batch_size_per_gpu: null
259
- forward_micro_batch_size: ${critic.ppo_micro_batch_size}
260
- forward_micro_batch_size_per_gpu: ${critic.ppo_micro_batch_size_per_gpu}
261
- use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz}
262
- ppo_max_token_len_per_gpu: 32768
263
- forward_max_token_len_per_gpu: ${critic.ppo_max_token_len_per_gpu}
264
- ulysses_sequence_parallel_size: 1
265
- ppo_epochs: ${actor_rollout_ref.actor.ppo_epochs}
266
- shuffle: ${actor_rollout_ref.actor.shuffle}
267
- grad_clip: 1.0
268
- cliprange_value: 0.5
269
- loss_agg_mode: ${actor_rollout_ref.actor.loss_agg_mode}
270
- checkpoint:
271
- contents:
272
- - model
273
- - optimizer
274
- - extra
275
- reward_model:
276
- enable: false
277
- strategy: fsdp
278
- model:
279
- input_tokenizer: ${actor_rollout_ref.model.path}
280
- path: ~/models/FsfairX-LLaMA3-RM-v0.1
281
- use_shm: false
282
- external_lib: ${actor_rollout_ref.model.external_lib}
283
- use_remove_padding: false
284
- use_fused_kernels: ${actor_rollout_ref.model.use_fused_kernels}
285
- trust_remote_code: false
286
- fsdp_config:
287
- wrap_policy:
288
- min_num_params: 0
289
- param_offload: false
290
- reshard_after_forward: true
291
- fsdp_size: -1
292
- micro_batch_size: null
293
- micro_batch_size_per_gpu: null
294
- max_length: null
295
- ulysses_sequence_parallel_size: 1
296
- use_dynamic_bsz: ${critic.use_dynamic_bsz}
297
- forward_max_token_len_per_gpu: ${critic.forward_max_token_len_per_gpu}
298
- reward_manager: naive
299
- launch_reward_fn_async: false
300
- sandbox_fusion:
301
- url: null
302
- max_concurrent: 64
303
- custom_reward_function:
304
- path: /scratch/user/sushil22_tamu.edu/projects/TreeHCA/TreeHCA/verl_treehca/verl/utils/reward_score/deep_research_em.py
305
- name: compute_score
306
- trainer:
307
- balance_batch: true
308
- total_epochs: 2
309
- total_training_steps: null
310
- project_name: Neurips_Workshop_2026
311
- experiment_name: TreeGRPO_hotpotqa_qwen4b_seed42_20260822_095129
312
- logger:
313
- - console
314
- - wandb
315
- log_val_generations: 0
316
- rollout_data_dir: /scratch/user/sushil22_tamu.edu/projects/TreeHCA/logs/TreeGRPO_hotpotqa_qwen4b/20260822_095129//rollout
317
- validation_data_dir: /scratch/user/sushil22_tamu.edu/projects/TreeHCA/logs/TreeGRPO_hotpotqa_qwen4b/20260822_095129//validation
318
- nnodes: 1
319
- n_gpus_per_node: 8
320
- save_freq: 250
321
- resume_mode: auto
322
- resume_from_path: null
323
- val_before_train: true
324
- test_freq: 10
325
- critic_warmup: 0
326
- default_hdfs_dir: null
327
- del_local_ckpt_after_load: false
328
- default_local_dir: /scratch/user/sushil22_tamu.edu/projects/TreeHCA/logs/TreeGRPO_hotpotqa_qwen4b/20260822_095129/
329
- max_actor_ckpt_to_keep: 1
330
- max_critic_ckpt_to_keep: null
331
- ray_wait_register_center_timeout: 300
332
- device: cuda
333
- ray_init:
334
- num_cpus: null
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
20260822_095129/outputs/.hydra/hydra.yaml DELETED
@@ -1,232 +0,0 @@
1
- hydra:
2
- run:
3
- dir: /scratch/user/sushil22_tamu.edu/projects/TreeHCA/logs/TreeGRPO_hotpotqa_qwen4b/20260822_095129//outputs
4
- sweep:
5
- dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
6
- subdir: ${hydra.job.num}
7
- launcher:
8
- _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
9
- sweeper:
10
- _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
11
- max_batch_size: null
12
- params: null
13
- help:
14
- app_name: ${hydra.job.name}
15
- header: '${hydra.help.app_name} is powered by Hydra.
16
-
17
- '
18
- footer: 'Powered by Hydra (https://hydra.cc)
19
-
20
- Use --hydra-help to view Hydra specific help
21
-
22
- '
23
- template: '${hydra.help.header}
24
-
25
- == Configuration groups ==
26
-
27
- Compose your configuration from those groups (group=option)
28
-
29
-
30
- $APP_CONFIG_GROUPS
31
-
32
-
33
- == Config ==
34
-
35
- Override anything in the config (foo.bar=value)
36
-
37
-
38
- $CONFIG
39
-
40
-
41
- ${hydra.help.footer}
42
-
43
- '
44
- hydra_help:
45
- template: 'Hydra (${hydra.runtime.version})
46
-
47
- See https://hydra.cc for more info.
48
-
49
-
50
- == Flags ==
51
-
52
- $FLAGS_HELP
53
-
54
-
55
- == Configuration groups ==
56
-
57
- Compose your configuration from those groups (For example, append hydra/job_logging=disabled
58
- to command line)
59
-
60
-
61
- $HYDRA_CONFIG_GROUPS
62
-
63
-
64
- Use ''--cfg hydra'' to Show the Hydra config.
65
-
66
- '
67
- hydra_help: ???
68
- hydra_logging:
69
- version: 1
70
- formatters:
71
- simple:
72
- format: '[%(asctime)s][HYDRA] %(message)s'
73
- handlers:
74
- console:
75
- class: logging.StreamHandler
76
- formatter: simple
77
- stream: ext://sys.stdout
78
- root:
79
- level: INFO
80
- handlers:
81
- - console
82
- loggers:
83
- logging_example:
84
- level: DEBUG
85
- disable_existing_loggers: false
86
- job_logging:
87
- version: 1
88
- formatters:
89
- simple:
90
- format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
91
- handlers:
92
- console:
93
- class: logging.StreamHandler
94
- formatter: simple
95
- stream: ext://sys.stdout
96
- file:
97
- class: logging.FileHandler
98
- formatter: simple
99
- filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
100
- root:
101
- level: INFO
102
- handlers:
103
- - console
104
- - file
105
- disable_existing_loggers: false
106
- env: {}
107
- mode: RUN
108
- searchpath: []
109
- callbacks: {}
110
- output_subdir: .hydra
111
- overrides:
112
- hydra:
113
- - hydra.run.dir=/scratch/user/sushil22_tamu.edu/projects/TreeHCA/logs/TreeGRPO_hotpotqa_qwen4b/20260822_095129//outputs
114
- - hydra.mode=RUN
115
- task:
116
- - algorithm.kl_ctrl.kl_coef=0.0
117
- - data.train_files=/scratch/user/sushil22_tamu.edu/projects/TreeHCA/rl_datasets/hotpotqa/train.parquet
118
- - data.val_files=["/scratch/user/sushil22_tamu.edu/projects/TreeHCA/rl_datasets/hotpotqa/test_512.parquet"]
119
- - data.prompt_key=prompt
120
- - data.train_batch_size=64
121
- - data.max_prompt_length=2000
122
- - data.max_response_length=6192
123
- - actor_rollout_ref.model.path=/scratch/user/sushil22_tamu.edu/projects/TreeHCA/models/Qwen3-4B
124
- - actor_rollout_ref.model.enable_gradient_checkpointing=True
125
- - actor_rollout_ref.model.use_remove_padding=True
126
- - actor_rollout_ref.actor.optim.lr=1e-6
127
- - actor_rollout_ref.actor.use_kl_loss=True
128
- - actor_rollout_ref.actor.kl_loss_coef=0.0
129
- - actor_rollout_ref.actor.kl_loss_type=low_var_kl
130
- - actor_rollout_ref.actor.fsdp_config.param_offload=False
131
- - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False
132
- - actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=32768
133
- - actor_rollout_ref.ref.fsdp_config.param_offload=True
134
- - trainer.critic_warmup=0
135
- - trainer.logger=[console, wandb]
136
- - trainer.project_name=Neurips_Workshop_2026
137
- - trainer.experiment_name=TreeGRPO_hotpotqa_qwen4b_seed42_20260822_095129
138
- - trainer.n_gpus_per_node=8
139
- - trainer.nnodes=1
140
- - trainer.save_freq=250
141
- - trainer.test_freq=10
142
- - trainer.total_epochs=2
143
- - trainer.default_local_dir=/scratch/user/sushil22_tamu.edu/projects/TreeHCA/logs/TreeGRPO_hotpotqa_qwen4b/20260822_095129/
144
- - trainer.val_before_train=True
145
- - ray_init.num_cpus=null
146
- - ++data.seed=42
147
- - algorithm.adv_estimator=grpo
148
- - actor_rollout_ref.actor.ppo_mini_batch_size=8
149
- - actor_rollout_ref.actor.use_dynamic_bsz=True
150
- - actor_rollout_ref.actor.ppo_max_token_len_per_gpu=16384
151
- - actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=32768
152
- - actor_rollout_ref.rollout.tensor_model_parallel_size=1
153
- - actor_rollout_ref.rollout.name=vllm
154
- - actor_rollout_ref.rollout.mode=sync_with_tool_tree_treegrpo
155
- - actor_rollout_ref.rollout.gpu_memory_utilization=0.6
156
- - actor_rollout_ref.rollout.n=22
157
- - actor_rollout_ref.rollout.initial_rollouts=10
158
- - actor_rollout_ref.rollout.beam_size=6
159
- - actor_rollout_ref.rollout.branch_probability=0.5
160
- - actor_rollout_ref.rollout.entropy_weight=0.2
161
- - ++actor_rollout_ref.rollout.reduce_gt_prob_context=False
162
- - actor_rollout_ref.rollout.tools.call_limit=6
163
- - ++actor_rollout_ref.rollout.tools.tool_instances.python.params.conda_path=/scratch/user/sushil22_tamu.edu/miniconda3
164
- - ++actor_rollout_ref.rollout.tools.tool_instances.python.params.conda_env=treehca_env
165
- - ++actor_rollout_ref.rollout.tools.tool_instances.search.params.cache_file=null
166
- - ++actor_rollout_ref.rollout.tools.tool_instances.search.params.api_key=unused_local_rag_server
167
- - actor_rollout_ref.rollout.multi_turn.enable=True
168
- - actor_rollout_ref.rollout.multi_turn.tool_config_path=/scratch/user/sushil22_tamu.edu/projects/TreeHCA/TreeHCA/verl_treehca/examples/sglang_multiturn/config/tool_config/search_tool_config.yaml
169
- - reward_model.reward_manager=naive
170
- - custom_reward_function.path=/scratch/user/sushil22_tamu.edu/projects/TreeHCA/TreeHCA/verl_treehca/verl/utils/reward_score/deep_research_em.py
171
- - custom_reward_function.name=compute_score
172
- - trainer.max_actor_ckpt_to_keep=1
173
- - +actor_rollout_ref.actor.checkpoint.save_contents=["hf_model","extra"]
174
- - trainer.rollout_data_dir=/scratch/user/sushil22_tamu.edu/projects/TreeHCA/logs/TreeGRPO_hotpotqa_qwen4b/20260822_095129//rollout
175
- - trainer.validation_data_dir=/scratch/user/sushil22_tamu.edu/projects/TreeHCA/logs/TreeGRPO_hotpotqa_qwen4b/20260822_095129//validation
176
- - actor_rollout_ref.actor.policy_loss=grpo
177
- - actor_rollout_ref.rollout.expansion_mode=random
178
- - actor_rollout_ref.rollout.initial_rollouts=1
179
- - actor_rollout_ref.rollout.beam_size=10
180
- - actor_rollout_ref.rollout.expansion_iterations=1
181
- - actor_rollout_ref.rollout.samples_per_tree=11
182
- - actor_rollout_ref.rollout.n=22
183
- - +actor_rollout_ref.rollout.num_trees_per_prompt=2
184
- - +actor_rollout_ref.rollout.reward_mode=base
185
- - +actor_rollout_ref.rollout.compute_gt_prob_stats=False
186
- - actor_rollout_ref.rollout.leaf_value_source=raw
187
- - actor_rollout_ref.rollout.leaf_value_norm=False
188
- - actor_rollout_ref.rollout.rescue_expansion_threshold=0.0
189
- - actor_rollout_ref.rollout.prune_incorrect_first=False
190
- - actor_rollout_ref.rollout.score_leaves_on_finish=False
191
- - actor_rollout_ref.rollout.enable_dynamic_rollouts=False
192
- job:
193
- name: main_ppo
194
- chdir: null
195
- override_dirname: ++actor_rollout_ref.rollout.reduce_gt_prob_context=False,++actor_rollout_ref.rollout.tools.tool_instances.python.params.conda_env=treehca_env,++actor_rollout_ref.rollout.tools.tool_instances.python.params.conda_path=/scratch/user/sushil22_tamu.edu/miniconda3,++actor_rollout_ref.rollout.tools.tool_instances.search.params.api_key=unused_local_rag_server,++actor_rollout_ref.rollout.tools.tool_instances.search.params.cache_file=null,++data.seed=42,+actor_rollout_ref.actor.checkpoint.save_contents=["hf_model","extra"],+actor_rollout_ref.rollout.compute_gt_prob_stats=False,+actor_rollout_ref.rollout.num_trees_per_prompt=2,+actor_rollout_ref.rollout.reward_mode=base,actor_rollout_ref.actor.fsdp_config.optimizer_offload=False,actor_rollout_ref.actor.fsdp_config.param_offload=False,actor_rollout_ref.actor.kl_loss_coef=0.0,actor_rollout_ref.actor.kl_loss_type=low_var_kl,actor_rollout_ref.actor.optim.lr=1e-6,actor_rollout_ref.actor.policy_loss=grpo,actor_rollout_ref.actor.ppo_max_token_len_per_gpu=16384,actor_rollout_ref.actor.ppo_mini_batch_size=8,actor_rollout_ref.actor.use_dynamic_bsz=True,actor_rollout_ref.actor.use_kl_loss=True,actor_rollout_ref.model.enable_gradient_checkpointing=True,actor_rollout_ref.model.path=/scratch/user/sushil22_tamu.edu/projects/TreeHCA/models/Qwen3-4B,actor_rollout_ref.model.use_remove_padding=True,actor_rollout_ref.ref.fsdp_config.param_offload=True,actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=32768,actor_rollout_ref.rollout.beam_size=10,actor_rollout_ref.rollout.beam_size=6,actor_rollout_ref.rollout.branch_probability=0.5,actor_rollout_ref.rollout.enable_dynamic_rollouts=False,actor_rollout_ref.rollout.entropy_weight=0.2,actor_rollout_ref.rollout.expansion_iterations=1,actor_rollout_ref.rollout.expansion_mode=random,actor_rollout_ref.rollout.gpu_memory_utilization=0.6,actor_rollout_ref.rollout.initial_rollouts=1,actor_rollout_ref.rollout.initial_rollouts=10,actor_rollout_ref.rollout.leaf_value_norm=False,actor_rollout_ref.rollout.leaf_value_source=raw,actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=32768,actor_rollout_ref.rollout.mode=sync_with_tool_tree_treegrpo,actor_rollout_ref.rollout.multi_turn.enable=True,actor_rollout_ref.rollout.multi_turn.tool_config_path=/scratch/user/sushil22_tamu.edu/projects/TreeHCA/TreeHCA/verl_treehca/examples/sglang_multiturn/config/tool_config/search_tool_config.yaml,actor_rollout_ref.rollout.n=22,actor_rollout_ref.rollout.n=22,actor_rollout_ref.rollout.name=vllm,actor_rollout_ref.rollout.prune_incorrect_first=False,actor_rollout_ref.rollout.rescue_expansion_threshold=0.0,actor_rollout_ref.rollout.samples_per_tree=11,actor_rollout_ref.rollout.score_leaves_on_finish=False,actor_rollout_ref.rollout.tensor_model_parallel_size=1,actor_rollout_ref.rollout.tools.call_limit=6,algorithm.adv_estimator=grpo,algorithm.kl_ctrl.kl_coef=0.0,custom_reward_function.name=compute_score,custom_reward_function.path=/scratch/user/sushil22_tamu.edu/projects/TreeHCA/TreeHCA/verl_treehca/verl/utils/reward_score/deep_research_em.py,data.max_prompt_length=2000,data.max_response_length=6192,data.prompt_key=prompt,data.train_batch_size=64,data.train_files=/scratch/user/sushil22_tamu.edu/projects/TreeHCA/rl_datasets/hotpotqa/train.parquet,data.val_files=["/scratch/user/sushil22_tamu.edu/projects/TreeHCA/rl_datasets/hotpotqa/test_512.parquet"],ray_init.num_cpus=null,reward_model.reward_manager=naive,trainer.critic_warmup=0,trainer.default_local_dir=/scratch/user/sushil22_tamu.edu/projects/TreeHCA/logs/TreeGRPO_hotpotqa_qwen4b/20260822_095129/,trainer.experiment_name=TreeGRPO_hotpotqa_qwen4b_seed42_20260822_095129,trainer.logger=[console,
196
- wandb],trainer.max_actor_ckpt_to_keep=1,trainer.n_gpus_per_node=8,trainer.nnodes=1,trainer.project_name=Neurips_Workshop_2026,trainer.rollout_data_dir=/scratch/user/sushil22_tamu.edu/projects/TreeHCA/logs/TreeGRPO_hotpotqa_qwen4b/20260822_095129//rollout,trainer.save_freq=250,trainer.test_freq=10,trainer.total_epochs=2,trainer.val_before_train=True,trainer.validation_data_dir=/scratch/user/sushil22_tamu.edu/projects/TreeHCA/logs/TreeGRPO_hotpotqa_qwen4b/20260822_095129//validation
197
- id: ???
198
- num: ???
199
- config_name: ppo_trainer_dr_turn.yaml
200
- env_set: {}
201
- env_copy: []
202
- config:
203
- override_dirname:
204
- kv_sep: '='
205
- item_sep: ','
206
- exclude_keys: []
207
- runtime:
208
- version: 1.3.2
209
- version_base: '1.3'
210
- cwd: /scratch/user/sushil22_tamu.edu/projects/TreeHCA
211
- config_sources:
212
- - path: hydra.conf
213
- schema: pkg
214
- provider: hydra
215
- - path: /scratch/user/sushil22_tamu.edu/projects/TreeHCA/TreeHCA/config
216
- schema: file
217
- provider: main
218
- - path: ''
219
- schema: structured
220
- provider: schema
221
- output_dir: /scratch/user/sushil22_tamu.edu/projects/TreeHCA/logs/TreeGRPO_hotpotqa_qwen4b/20260822_095129/outputs
222
- choices:
223
- hydra/env: default
224
- hydra/callbacks: null
225
- hydra/job_logging: default
226
- hydra/hydra_logging: default
227
- hydra/hydra_help: default
228
- hydra/help: default
229
- hydra/sweeper: basic
230
- hydra/launcher: basic
231
- hydra/output: default
232
- verbose: false
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
20260822_095129/outputs/.hydra/overrides.yaml DELETED
@@ -1,76 +0,0 @@
1
- - algorithm.kl_ctrl.kl_coef=0.0
2
- - data.train_files=/scratch/user/sushil22_tamu.edu/projects/TreeHCA/rl_datasets/hotpotqa/train.parquet
3
- - data.val_files=["/scratch/user/sushil22_tamu.edu/projects/TreeHCA/rl_datasets/hotpotqa/test_512.parquet"]
4
- - data.prompt_key=prompt
5
- - data.train_batch_size=64
6
- - data.max_prompt_length=2000
7
- - data.max_response_length=6192
8
- - actor_rollout_ref.model.path=/scratch/user/sushil22_tamu.edu/projects/TreeHCA/models/Qwen3-4B
9
- - actor_rollout_ref.model.enable_gradient_checkpointing=True
10
- - actor_rollout_ref.model.use_remove_padding=True
11
- - actor_rollout_ref.actor.optim.lr=1e-6
12
- - actor_rollout_ref.actor.use_kl_loss=True
13
- - actor_rollout_ref.actor.kl_loss_coef=0.0
14
- - actor_rollout_ref.actor.kl_loss_type=low_var_kl
15
- - actor_rollout_ref.actor.fsdp_config.param_offload=False
16
- - actor_rollout_ref.actor.fsdp_config.optimizer_offload=False
17
- - actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=32768
18
- - actor_rollout_ref.ref.fsdp_config.param_offload=True
19
- - trainer.critic_warmup=0
20
- - trainer.logger=[console, wandb]
21
- - trainer.project_name=Neurips_Workshop_2026
22
- - trainer.experiment_name=TreeGRPO_hotpotqa_qwen4b_seed42_20260822_095129
23
- - trainer.n_gpus_per_node=8
24
- - trainer.nnodes=1
25
- - trainer.save_freq=250
26
- - trainer.test_freq=10
27
- - trainer.total_epochs=2
28
- - trainer.default_local_dir=/scratch/user/sushil22_tamu.edu/projects/TreeHCA/logs/TreeGRPO_hotpotqa_qwen4b/20260822_095129/
29
- - trainer.val_before_train=True
30
- - ray_init.num_cpus=null
31
- - ++data.seed=42
32
- - algorithm.adv_estimator=grpo
33
- - actor_rollout_ref.actor.ppo_mini_batch_size=8
34
- - actor_rollout_ref.actor.use_dynamic_bsz=True
35
- - actor_rollout_ref.actor.ppo_max_token_len_per_gpu=16384
36
- - actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=32768
37
- - actor_rollout_ref.rollout.tensor_model_parallel_size=1
38
- - actor_rollout_ref.rollout.name=vllm
39
- - actor_rollout_ref.rollout.mode=sync_with_tool_tree_treegrpo
40
- - actor_rollout_ref.rollout.gpu_memory_utilization=0.6
41
- - actor_rollout_ref.rollout.n=22
42
- - actor_rollout_ref.rollout.initial_rollouts=10
43
- - actor_rollout_ref.rollout.beam_size=6
44
- - actor_rollout_ref.rollout.branch_probability=0.5
45
- - actor_rollout_ref.rollout.entropy_weight=0.2
46
- - ++actor_rollout_ref.rollout.reduce_gt_prob_context=False
47
- - actor_rollout_ref.rollout.tools.call_limit=6
48
- - ++actor_rollout_ref.rollout.tools.tool_instances.python.params.conda_path=/scratch/user/sushil22_tamu.edu/miniconda3
49
- - ++actor_rollout_ref.rollout.tools.tool_instances.python.params.conda_env=treehca_env
50
- - ++actor_rollout_ref.rollout.tools.tool_instances.search.params.cache_file=null
51
- - ++actor_rollout_ref.rollout.tools.tool_instances.search.params.api_key=unused_local_rag_server
52
- - actor_rollout_ref.rollout.multi_turn.enable=True
53
- - actor_rollout_ref.rollout.multi_turn.tool_config_path=/scratch/user/sushil22_tamu.edu/projects/TreeHCA/TreeHCA/verl_treehca/examples/sglang_multiturn/config/tool_config/search_tool_config.yaml
54
- - reward_model.reward_manager=naive
55
- - custom_reward_function.path=/scratch/user/sushil22_tamu.edu/projects/TreeHCA/TreeHCA/verl_treehca/verl/utils/reward_score/deep_research_em.py
56
- - custom_reward_function.name=compute_score
57
- - trainer.max_actor_ckpt_to_keep=1
58
- - +actor_rollout_ref.actor.checkpoint.save_contents=["hf_model","extra"]
59
- - trainer.rollout_data_dir=/scratch/user/sushil22_tamu.edu/projects/TreeHCA/logs/TreeGRPO_hotpotqa_qwen4b/20260822_095129//rollout
60
- - trainer.validation_data_dir=/scratch/user/sushil22_tamu.edu/projects/TreeHCA/logs/TreeGRPO_hotpotqa_qwen4b/20260822_095129//validation
61
- - actor_rollout_ref.actor.policy_loss=grpo
62
- - actor_rollout_ref.rollout.expansion_mode=random
63
- - actor_rollout_ref.rollout.initial_rollouts=1
64
- - actor_rollout_ref.rollout.beam_size=10
65
- - actor_rollout_ref.rollout.expansion_iterations=1
66
- - actor_rollout_ref.rollout.samples_per_tree=11
67
- - actor_rollout_ref.rollout.n=22
68
- - +actor_rollout_ref.rollout.num_trees_per_prompt=2
69
- - +actor_rollout_ref.rollout.reward_mode=base
70
- - +actor_rollout_ref.rollout.compute_gt_prob_stats=False
71
- - actor_rollout_ref.rollout.leaf_value_source=raw
72
- - actor_rollout_ref.rollout.leaf_value_norm=False
73
- - actor_rollout_ref.rollout.rescue_expansion_threshold=0.0
74
- - actor_rollout_ref.rollout.prune_incorrect_first=False
75
- - actor_rollout_ref.rollout.score_leaves_on_finish=False
76
- - actor_rollout_ref.rollout.enable_dynamic_rollouts=False
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
20260822_095129/outputs/main_ppo.log DELETED
File without changes
20260822_095129/rollout/1.jsonl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:3cddc675ddfd3c1b2dee88f7f30734af7d4725744e3d7929a405717cfceafcd6
3
- size 18403147
 
 
 
 
20260822_095129/rollout/10.jsonl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:1c71a066ce8849e31f2c47b058dc0a17b75b6e90b867cf5fee4bc95b804c0525
3
- size 15316396
 
 
 
 
20260822_095129/rollout/101.jsonl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:a7ccc33d55fe0f263d34b33a4897234a2f5e8bea7005e781acdc2c11d85538b2
3
- size 13330112
 
 
 
 
20260822_095129/rollout/103.jsonl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:6b74b4d3156efdc64ca2dedab5925aef6991ca2bef17b2b53970712589399fe8
3
- size 14460567
 
 
 
 
20260822_095129/rollout/106.jsonl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:18483b3a49e95ff71471968832cf2dc9deb8b3930b694636df2c537dbdec105b
3
- size 13938292
 
 
 
 
20260822_095129/rollout/107.jsonl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:28c4369483a73e3b9f689029a5a0c93055cccf3042b38394191639d174a6bcb8
3
- size 14961466
 
 
 
 
20260822_095129/rollout/108.jsonl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:594d8ba50ebad9922cf84250e0b0a97fd754d78ceefc30a209dcdee9c8aa07d1
3
- size 13119068
 
 
 
 
20260822_095129/rollout/109.jsonl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:152a156474db4877247c7f55f69e5c960c478a26accf8c18944a78f28bcc375b
3
- size 13924998
 
 
 
 
20260822_095129/rollout/110.jsonl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:aa5e28cadfee0495e1020faf54fbd5a7700602718ce4d3c8463235a28c727704
3
- size 14163060
 
 
 
 
20260822_095129/rollout/112.jsonl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:b800c06634173bf51d63d53c7f19dac70697bddda83eb7a64ea9e9b78d11ee83
3
- size 13633713
 
 
 
 
20260822_095129/rollout/113.jsonl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:f5815ab0ce08474fbba95ddd559e1ade0dd73ae15198b178e02795c8d2a1c086
3
- size 14222144
 
 
 
 
20260822_095129/rollout/115.jsonl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:9a63c331cc972432ec884785303f7dd2cd35a83b353f472c6b14bd49badadcb8
3
- size 14935994
 
 
 
 
20260822_095129/rollout/116.jsonl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:7693a9bed24210734287c48726086cab1196d5438d8bd3fd251a54c02c74cb6c
3
- size 14239971
 
 
 
 
20260822_095129/rollout/118.jsonl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:bc765030a7cc8271ee01daa769acaf319e2f06738f561107803cf9c2eef10350
3
- size 12440371
 
 
 
 
20260822_095129/rollout/12.jsonl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:0aef9c3adfda5be4fb9eba5977edbfcb50b487ae0bc3220b6f96705f25c649d2
3
- size 14939263
 
 
 
 
20260822_095129/rollout/121.jsonl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:e131b7121cd63a906aa147e52039a9d17801c3cb10ec1cbdbb56e16b1f62a1c5
3
- size 14037381
 
 
 
 
20260822_095129/rollout/122.jsonl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:62a8ec60124413c03be9e7342a34ca6599c1e1a8141e3d22a6348bf92c34dd0a
3
- size 14662511
 
 
 
 
20260822_095129/rollout/123.jsonl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:f273c299341610ba47484ad991a2e8375aa923a451c559019592afed5b924e99
3
- size 14874985
 
 
 
 
20260822_095129/rollout/124.jsonl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:781a86f6d0820a9007f1a9e10eb9fea625fe4752f596288220eccf2832783063
3
- size 14011869
 
 
 
 
20260822_095129/rollout/129.jsonl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:24a03a3304ca92e437992019bd1cc0a90a69ba3dcf564dfde88eab67d3b7bce7
3
- size 13892858
 
 
 
 
20260822_095129/rollout/13.jsonl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:b00cc451a9d7b5c35ed98ce60e8c1698a731353390f18b59ca3036016e6637ed
3
- size 14460049
 
 
 
 
20260822_095129/rollout/130.jsonl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:c7e9e67ffd0f8fa0d38b68ec306527171e705b469c97528e8aa77f8288e27f3c
3
- size 13949791
 
 
 
 
20260822_095129/rollout/131.jsonl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:0a57c409a162785df6c3ddfd8fc178489231094cfb98e90668b72cbd79f81295
3
- size 14327170
 
 
 
 
20260822_095129/rollout/132.jsonl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:afbbd0bab2d1376be9d727b685e687ee1391b1633e11dffa7044f949f08764b4
3
- size 15035859
 
 
 
 
20260822_095129/rollout/133.jsonl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:98f33336294a4b92972dfc4b42b927272a94be8767256ae9cd62010f41ecce12
3
- size 14564596
 
 
 
 
20260822_095129/rollout/134.jsonl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:692987f84d789cafec765290f9a87c486a5a92f60b85dac09956b5959a2c823b
3
- size 13464432
 
 
 
 
20260822_095129/rollout/135.jsonl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:2e4afa76903b65a1a5116e605bd638627c55ec112c8305e2338277232087cba2
3
- size 13631158
 
 
 
 
20260822_095129/rollout/136.jsonl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:1daac20bf764891cae41495df37cfc546df4fe70b647d17c025048c4b6a52233
3
- size 14819000