Hugging Face's logo

JOY0021
/
autonomy-agent-v2

Transformers
Safetensors
Generated from Trainer
trl
grpo
Model card Files
xet
 Community
JOY0021 commited on
Commit
1c6eae6
·
verified ·
1 Parent(s): 2ce2593

Upload folder using huggingface_hub

Browse files
Files changed (24) hide show
  1. .gitattributes +2 -0
  2. README.md +67 -0
  3. checkpoint-25/chat_template.jinja +54 -0
  4. checkpoint-25/config.json +57 -0
  5. checkpoint-25/generation_config.json +13 -0
  6. checkpoint-25/model.safetensors +3 -0
  7. checkpoint-25/optimizer.pt +3 -0
  8. checkpoint-25/rng_state.pth +3 -0
  9. checkpoint-25/scheduler.pt +3 -0
  10. checkpoint-25/tokenizer.json +3 -0
  11. checkpoint-25/tokenizer_config.json +32 -0
  12. checkpoint-25/trainer_state.json +475 -0
  13. checkpoint-25/training_args.bin +3 -0
  14. checkpoint-50/chat_template.jinja +54 -0
  15. checkpoint-50/config.json +57 -0
  16. checkpoint-50/generation_config.json +13 -0
  17. checkpoint-50/model.safetensors +3 -0
  18. checkpoint-50/optimizer.pt +3 -0
  19. checkpoint-50/rng_state.pth +3 -0
  20. checkpoint-50/scheduler.pt +3 -0
  21. checkpoint-50/tokenizer.json +3 -0
  22. checkpoint-50/tokenizer_config.json +32 -0
  23. checkpoint-50/trainer_state.json +903 -0
  24. checkpoint-50/training_args.bin +3 -0
.gitattributes CHANGED
@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ checkpoint-25/tokenizer.json filter=lfs diff=lfs merge=lfs -text
37
+ checkpoint-50/tokenizer.json filter=lfs diff=lfs merge=lfs -text
README.md ADDED
@@ -0,0 +1,67 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ base_model: Qwen/Qwen2.5-0.5B-Instruct
3
+ library_name: transformers
4
+ model_name: autonomy-agent-v2
5
+ tags:
6
+ - generated_from_trainer
7
+ - trl
8
+ - grpo
9
+ licence: license
10
+ ---
11
+
12
+ # Model Card for autonomy-agent-v2
13
+
14
+ This model is a fine-tuned version of [Qwen/Qwen2.5-0.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct).
15
+ It has been trained using [TRL](https://github.com/huggingface/trl).
16
+
17
+ ## Quick start
18
+
19
+ ```python
20
+ from transformers import pipeline
21
+
22
+ question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
23
+ generator = pipeline("text-generation", model="None", device="cuda")
24
+ output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
25
+ print(output["generated_text"])
26
+ ```
27
+
28
+ ## Training procedure
29
+
30
+
31
+
32
+
33
+
34
+ This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).
35
+
36
+ ### Framework versions
37
+
38
+ - TRL: 1.2.0
39
+ - Transformers: 5.6.2
40
+ - Pytorch: 2.11.0
41
+ - Datasets: 4.8.4
42
+ - Tokenizers: 0.22.2
43
+
44
+ ## Citations
45
+
46
+ Cite GRPO as:
47
+
48
+ ```bibtex
49
+ @article{shao2024deepseekmath,
50
+ title = {{DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models}},
51
+ author = {Zhihong Shao and Peiyi Wang and Qihao Zhu and Runxin Xu and Junxiao Song and Mingchuan Zhang and Y. K. Li and Y. Wu and Daya Guo},
52
+ year = 2024,
53
+ eprint = {arXiv:2402.03300},
54
+ }
55
+ ```
56
+
57
+ Cite TRL as:
58
+
59
+ ```bibtex
60
+ @software{vonwerra2020trl,
61
+ title = {{TRL: Transformers Reinforcement Learning}},
62
+ author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
63
+ license = {Apache-2.0},
64
+ url = {https://github.com/huggingface/trl},
65
+ year = {2020}
66
+ }
67
+ ```
checkpoint-25/chat_template.jinja ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {%- if tools %}
2
+ {{- '<|im_start|>system\n' }}
3
+ {%- if messages[0]['role'] == 'system' %}
4
+ {{- messages[0]['content'] }}
5
+ {%- else %}
6
+ {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
7
+ {%- endif %}
8
+ {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
9
+ {%- for tool in tools %}
10
+ {{- "\n" }}
11
+ {{- tool | tojson }}
12
+ {%- endfor %}
13
+ {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
14
+ {%- else %}
15
+ {%- if messages[0]['role'] == 'system' %}
16
+ {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
17
+ {%- else %}
18
+ {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
19
+ {%- endif %}
20
+ {%- endif %}
21
+ {%- for message in messages %}
22
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
23
+ {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
24
+ {%- elif message.role == "assistant" %}
25
+ {{- '<|im_start|>' + message.role }}
26
+ {%- if message.content %}
27
+ {{- '\n' + message.content }}
28
+ {%- endif %}
29
+ {%- for tool_call in message.tool_calls %}
30
+ {%- if tool_call.function is defined %}
31
+ {%- set tool_call = tool_call.function %}
32
+ {%- endif %}
33
+ {{- '\n<tool_call>\n{"name": "' }}
34
+ {{- tool_call.name }}
35
+ {{- '", "arguments": ' }}
36
+ {{- tool_call.arguments | tojson }}
37
+ {{- '}\n</tool_call>' }}
38
+ {%- endfor %}
39
+ {{- '<|im_end|>\n' }}
40
+ {%- elif message.role == "tool" %}
41
+ {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
42
+ {{- '<|im_start|>user' }}
43
+ {%- endif %}
44
+ {{- '\n<tool_response>\n' }}
45
+ {{- message.content }}
46
+ {{- '\n</tool_response>' }}
47
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
48
+ {{- '<|im_end|>\n' }}
49
+ {%- endif %}
50
+ {%- endif %}
51
+ {%- endfor %}
52
+ {%- if add_generation_prompt %}
53
+ {{- '<|im_start|>assistant\n' }}
54
+ {%- endif %}
checkpoint-25/config.json ADDED
@@ -0,0 +1,57 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen2ForCausalLM"
4
+ ],
5
+ "attention_dropout": 0.0,
6
+ "bos_token_id": null,
7
+ "dtype": "bfloat16",
8
+ "eos_token_id": 151645,
9
+ "hidden_act": "silu",
10
+ "hidden_size": 896,
11
+ "initializer_range": 0.02,
12
+ "intermediate_size": 4864,
13
+ "layer_types": [
14
+ "full_attention",
15
+ "full_attention",
16
+ "full_attention",
17
+ "full_attention",
18
+ "full_attention",
19
+ "full_attention",
20
+ "full_attention",
21
+ "full_attention",
22
+ "full_attention",
23
+ "full_attention",
24
+ "full_attention",
25
+ "full_attention",
26
+ "full_attention",
27
+ "full_attention",
28
+ "full_attention",
29
+ "full_attention",
30
+ "full_attention",
31
+ "full_attention",
32
+ "full_attention",
33
+ "full_attention",
34
+ "full_attention",
35
+ "full_attention",
36
+ "full_attention",
37
+ "full_attention"
38
+ ],
39
+ "max_position_embeddings": 32768,
40
+ "max_window_layers": 21,
41
+ "model_type": "qwen2",
42
+ "num_attention_heads": 14,
43
+ "num_hidden_layers": 24,
44
+ "num_key_value_heads": 2,
45
+ "pad_token_id": 151643,
46
+ "rms_norm_eps": 1e-06,
47
+ "rope_parameters": {
48
+ "rope_theta": 1000000.0,
49
+ "rope_type": "default"
50
+ },
51
+ "sliding_window": null,
52
+ "tie_word_embeddings": true,
53
+ "transformers_version": "5.6.2",
54
+ "use_cache": false,
55
+ "use_sliding_window": false,
56
+ "vocab_size": 151936
57
+ }
checkpoint-25/generation_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "do_sample": true,
3
+ "eos_token_id": [
4
+ 151645,
5
+ 151643
6
+ ],
7
+ "pad_token_id": 151643,
8
+ "repetition_penalty": 1.1,
9
+ "temperature": 0.7,
10
+ "top_k": 20,
11
+ "top_p": 0.8,
12
+ "transformers_version": "5.6.2"
13
+ }
checkpoint-25/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fdf756fa7fcbe7404d5c60e26bff1a0c8b8aa1f72ced49e7dd0210fe288fb7fe
3
+ size 988097824
checkpoint-25/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:49070c7706b4d474935cf3ea7b357b2c28081a56fdb74d22bf333b8bc8bd6977
3
+ size 1976378699
checkpoint-25/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c26fb843dab8e5657e4e5578986260ff3eab826cf10a8afcb7340750033c35f7
3
+ size 14645
checkpoint-25/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7a5b3364c61da829cfa47305bd0597607812e658ee7f7017c30727fef04f7e9d
3
+ size 1465
checkpoint-25/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
3
+ size 11421892
checkpoint-25/tokenizer_config.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "backend": "tokenizers",
4
+ "bos_token": null,
5
+ "clean_up_tokenization_spaces": false,
6
+ "eos_token": "<|im_end|>",
7
+ "errors": "replace",
8
+ "extra_special_tokens": [
9
+ "<|im_start|>",
10
+ "<|im_end|>",
11
+ "<|object_ref_start|>",
12
+ "<|object_ref_end|>",
13
+ "<|box_start|>",
14
+ "<|box_end|>",
15
+ "<|quad_start|>",
16
+ "<|quad_end|>",
17
+ "<|vision_start|>",
18
+ "<|vision_end|>",
19
+ "<|vision_pad|>",
20
+ "<|image_pad|>",
21
+ "<|video_pad|>"
22
+ ],
23
+ "is_local": false,
24
+ "local_files_only": false,
25
+ "model_max_length": 131072,
26
+ "pad_token": "<|endoftext|>",
27
+ "padding_side": "left",
28
+ "split_special_tokens": false,
29
+ "tokenizer_class": "Qwen2Tokenizer",
30
+ "truncation_side": "left",
31
+ "unk_token": null
32
+ }
checkpoint-25/trainer_state.json ADDED
@@ -0,0 +1,475 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 0.0625,
6
+ "eval_steps": 500,
7
+ "global_step": 25,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "clip_ratio/high_max": 0.0,
14
+ "clip_ratio/high_mean": 0.0,
15
+ "clip_ratio/low_mean": 0.0,
16
+ "clip_ratio/low_min": 0.0,
17
+ "clip_ratio/region_mean": 0.0,
18
+ "completions/clipped_ratio": 1.0,
19
+ "completions/max_length": 256.0,
20
+ "completions/max_terminated_length": 0.0,
21
+ "completions/mean_length": 256.0,
22
+ "completions/mean_terminated_length": 0.0,
23
+ "completions/min_length": 256.0,
24
+ "completions/min_terminated_length": 0.0,
25
+ "entropy": 1.975826621055603,
26
+ "epoch": 0.0025,
27
+ "frac_reward_zero_std": 1.0,
28
+ "grad_norm": 0.0,
29
+ "learning_rate": 1e-05,
30
+ "loss": 0.0,
31
+ "num_tokens": 1132.0,
32
+ "reward": 0.009999999776482582,
33
+ "reward_std": 0.0,
34
+ "rewards/autonomy_reward_fn/mean": 0.009999999776482582,
35
+ "rewards/autonomy_reward_fn/std": 0.0,
36
+ "step": 1,
37
+ "step_time": 5.940139313000145
38
+ },
39
+ {
40
+ "clip_ratio/high_max": 0.0,
41
+ "clip_ratio/high_mean": 0.0,
42
+ "clip_ratio/low_mean": 0.0,
43
+ "clip_ratio/low_min": 0.0,
44
+ "clip_ratio/region_mean": 0.0,
45
+ "entropy": 2.439243793487549,
46
+ "epoch": 0.005,
47
+ "grad_norm": 0.0,
48
+ "learning_rate": 9.800000000000001e-06,
49
+ "loss": 0.0,
50
+ "step": 2,
51
+ "step_time": 0.13269777900040935
52
+ },
53
+ {
54
+ "clip_ratio/high_max": 0.0,
55
+ "clip_ratio/high_mean": 0.0,
56
+ "clip_ratio/low_mean": 0.0,
57
+ "clip_ratio/low_min": 0.0,
58
+ "clip_ratio/region_mean": 0.0,
59
+ "entropy": 2.167536497116089,
60
+ "epoch": 0.0075,
61
+ "grad_norm": 0.0,
62
+ "learning_rate": 9.600000000000001e-06,
63
+ "loss": 0.0,
64
+ "step": 3,
65
+ "step_time": 0.12384193100024277
66
+ },
67
+ {
68
+ "clip_ratio/high_max": 0.0,
69
+ "clip_ratio/high_mean": 0.0,
70
+ "clip_ratio/low_mean": 0.0,
71
+ "clip_ratio/low_min": 0.0,
72
+ "clip_ratio/region_mean": 0.0,
73
+ "entropy": 2.270764112472534,
74
+ "epoch": 0.01,
75
+ "grad_norm": 0.0,
76
+ "learning_rate": 9.4e-06,
77
+ "loss": 0.0,
78
+ "step": 4,
79
+ "step_time": 0.12020948200006387
80
+ },
81
+ {
82
+ "clip_ratio/high_max": 0.0,
83
+ "clip_ratio/high_mean": 0.0,
84
+ "clip_ratio/low_mean": 0.0,
85
+ "clip_ratio/low_min": 0.0,
86
+ "clip_ratio/region_mean": 0.0,
87
+ "completions/clipped_ratio": 1.0,
88
+ "completions/max_length": 256.0,
89
+ "completions/max_terminated_length": 0.0,
90
+ "completions/mean_length": 256.0,
91
+ "completions/mean_terminated_length": 0.0,
92
+ "completions/min_length": 256.0,
93
+ "completions/min_terminated_length": 0.0,
94
+ "entropy": 1.7375574111938477,
95
+ "epoch": 0.0125,
96
+ "frac_reward_zero_std": 1.0,
97
+ "grad_norm": 0.0,
98
+ "learning_rate": 9.200000000000002e-06,
99
+ "loss": 0.0,
100
+ "num_tokens": 2240.0,
101
+ "reward": 0.009999999776482582,
102
+ "reward_std": 0.0,
103
+ "rewards/autonomy_reward_fn/mean": 0.009999999776482582,
104
+ "rewards/autonomy_reward_fn/std": 0.0,
105
+ "step": 5,
106
+ "step_time": 5.4328740460005065
107
+ },
108
+ {
109
+ "clip_ratio/high_max": 0.0,
110
+ "clip_ratio/high_mean": 0.0,
111
+ "clip_ratio/low_mean": 0.0,
112
+ "clip_ratio/low_min": 0.0,
113
+ "clip_ratio/region_mean": 0.0,
114
+ "entropy": 1.2307178974151611,
115
+ "epoch": 0.015,
116
+ "grad_norm": 0.0,
117
+ "learning_rate": 9e-06,
118
+ "loss": 0.0,
119
+ "step": 6,
120
+ "step_time": 0.12100409000049694
121
+ },
122
+ {
123
+ "clip_ratio/high_max": 0.0,
124
+ "clip_ratio/high_mean": 0.0,
125
+ "clip_ratio/low_mean": 0.0,
126
+ "clip_ratio/low_min": 0.0,
127
+ "clip_ratio/region_mean": 0.0,
128
+ "entropy": 2.5965301990509033,
129
+ "epoch": 0.0175,
130
+ "grad_norm": 0.0,
131
+ "learning_rate": 8.8e-06,
132
+ "loss": 0.0,
133
+ "step": 7,
134
+ "step_time": 0.11992437099979725
135
+ },
136
+ {
137
+ "clip_ratio/high_max": 0.0,
138
+ "clip_ratio/high_mean": 0.0,
139
+ "clip_ratio/low_mean": 0.0,
140
+ "clip_ratio/low_min": 0.0,
141
+ "clip_ratio/region_mean": 0.0,
142
+ "entropy": 3.000967502593994,
143
+ "epoch": 0.02,
144
+ "grad_norm": 0.0,
145
+ "learning_rate": 8.6e-06,
146
+ "loss": 0.0,
147
+ "step": 8,
148
+ "step_time": 0.12026366800000687
149
+ },
150
+ {
151
+ "clip_ratio/high_max": 0.0,
152
+ "clip_ratio/high_mean": 0.0,
153
+ "clip_ratio/low_mean": 0.0,
154
+ "clip_ratio/low_min": 0.0,
155
+ "clip_ratio/region_mean": 0.0,
156
+ "completions/clipped_ratio": 1.0,
157
+ "completions/max_length": 256.0,
158
+ "completions/max_terminated_length": 0.0,
159
+ "completions/mean_length": 256.0,
160
+ "completions/mean_terminated_length": 0.0,
161
+ "completions/min_length": 256.0,
162
+ "completions/min_terminated_length": 0.0,
163
+ "entropy": 2.1747303009033203,
164
+ "epoch": 0.0225,
165
+ "frac_reward_zero_std": 1.0,
166
+ "grad_norm": 0.0,
167
+ "learning_rate": 8.400000000000001e-06,
168
+ "loss": 0.0,
169
+ "num_tokens": 3372.0,
170
+ "reward": 0.009999999776482582,
171
+ "reward_std": 0.0,
172
+ "rewards/autonomy_reward_fn/mean": 0.009999999776482582,
173
+ "rewards/autonomy_reward_fn/std": 0.0,
174
+ "step": 9,
175
+ "step_time": 5.488657728999897
176
+ },
177
+ {
178
+ "clip_ratio/high_max": 0.0,
179
+ "clip_ratio/high_mean": 0.0,
180
+ "clip_ratio/low_mean": 0.0,
181
+ "clip_ratio/low_min": 0.0,
182
+ "clip_ratio/region_mean": 0.0,
183
+ "entropy": 2.2987818717956543,
184
+ "epoch": 0.025,
185
+ "grad_norm": 0.0,
186
+ "learning_rate": 8.2e-06,
187
+ "loss": 0.0,
188
+ "step": 10,
189
+ "step_time": 0.12148796400015272
190
+ },
191
+ {
192
+ "clip_ratio/high_max": 0.0,
193
+ "clip_ratio/high_mean": 0.0,
194
+ "clip_ratio/low_mean": 0.0,
195
+ "clip_ratio/low_min": 0.0,
196
+ "clip_ratio/region_mean": 0.0,
197
+ "entropy": 2.549589157104492,
198
+ "epoch": 0.0275,
199
+ "grad_norm": 0.0,
200
+ "learning_rate": 8.000000000000001e-06,
201
+ "loss": 0.0,
202
+ "step": 11,
203
+ "step_time": 0.1202768709999873
204
+ },
205
+ {
206
+ "clip_ratio/high_max": 0.0,
207
+ "clip_ratio/high_mean": 0.0,
208
+ "clip_ratio/low_mean": 0.0,
209
+ "clip_ratio/low_min": 0.0,
210
+ "clip_ratio/region_mean": 0.0,
211
+ "entropy": 2.3748786449432373,
212
+ "epoch": 0.03,
213
+ "grad_norm": 0.0,
214
+ "learning_rate": 7.800000000000002e-06,
215
+ "loss": 0.0,
216
+ "step": 12,
217
+ "step_time": 0.12146498200036149
218
+ },
219
+ {
220
+ "clip_ratio/high_max": 0.0,
221
+ "clip_ratio/high_mean": 0.0,
222
+ "clip_ratio/low_mean": 0.0,
223
+ "clip_ratio/low_min": 0.0,
224
+ "clip_ratio/region_mean": 0.0,
225
+ "completions/clipped_ratio": 1.0,
226
+ "completions/max_length": 256.0,
227
+ "completions/max_terminated_length": 0.0,
228
+ "completions/mean_length": 256.0,
229
+ "completions/mean_terminated_length": 0.0,
230
+ "completions/min_length": 256.0,
231
+ "completions/min_terminated_length": 0.0,
232
+ "entropy": 2.562595844268799,
233
+ "epoch": 0.0325,
234
+ "frac_reward_zero_std": 1.0,
235
+ "grad_norm": 0.0,
236
+ "learning_rate": 7.600000000000001e-06,
237
+ "loss": 0.0,
238
+ "num_tokens": 4504.0,
239
+ "reward": 0.009999999776482582,
240
+ "reward_std": 0.0,
241
+ "rewards/autonomy_reward_fn/mean": 0.009999999776482582,
242
+ "rewards/autonomy_reward_fn/std": 0.0,
243
+ "step": 13,
244
+ "step_time": 5.529226956000457
245
+ },
246
+ {
247
+ "clip_ratio/high_max": 0.0,
248
+ "clip_ratio/high_mean": 0.0,
249
+ "clip_ratio/low_mean": 0.0,
250
+ "clip_ratio/low_min": 0.0,
251
+ "clip_ratio/region_mean": 0.0,
252
+ "entropy": 2.9105567932128906,
253
+ "epoch": 0.035,
254
+ "grad_norm": 0.0,
255
+ "learning_rate": 7.4e-06,
256
+ "loss": 0.0,
257
+ "step": 14,
258
+ "step_time": 0.12003547699987394
259
+ },
260
+ {
261
+ "clip_ratio/high_max": 0.0,
262
+ "clip_ratio/high_mean": 0.0,
263
+ "clip_ratio/low_mean": 0.0,
264
+ "clip_ratio/low_min": 0.0,
265
+ "clip_ratio/region_mean": 0.0,
266
+ "entropy": 2.360198497772217,
267
+ "epoch": 0.0375,
268
+ "grad_norm": 0.0,
269
+ "learning_rate": 7.2000000000000005e-06,
270
+ "loss": 0.0,
271
+ "step": 15,
272
+ "step_time": 0.12056965900046634
273
+ },
274
+ {
275
+ "clip_ratio/high_max": 0.0,
276
+ "clip_ratio/high_mean": 0.0,
277
+ "clip_ratio/low_mean": 0.0,
278
+ "clip_ratio/low_min": 0.0,
279
+ "clip_ratio/region_mean": 0.0,
280
+ "entropy": 1.9906266927719116,
281
+ "epoch": 0.04,
282
+ "grad_norm": 0.0,
283
+ "learning_rate": 7e-06,
284
+ "loss": 0.0,
285
+ "step": 16,
286
+ "step_time": 0.11991975299952173
287
+ },
288
+ {
289
+ "clip_ratio/high_max": 0.0,
290
+ "clip_ratio/high_mean": 0.0,
291
+ "clip_ratio/low_mean": 0.0,
292
+ "clip_ratio/low_min": 0.0,
293
+ "clip_ratio/region_mean": 0.0,
294
+ "completions/clipped_ratio": 1.0,
295
+ "completions/max_length": 256.0,
296
+ "completions/max_terminated_length": 0.0,
297
+ "completions/mean_length": 256.0,
298
+ "completions/mean_terminated_length": 0.0,
299
+ "completions/min_length": 256.0,
300
+ "completions/min_terminated_length": 0.0,
301
+ "entropy": 2.4847490787506104,
302
+ "epoch": 0.0425,
303
+ "frac_reward_zero_std": 1.0,
304
+ "grad_norm": 0.0,
305
+ "learning_rate": 6.800000000000001e-06,
306
+ "loss": 0.0,
307
+ "num_tokens": 5636.0,
308
+ "reward": 0.009999999776482582,
309
+ "reward_std": 0.0,
310
+ "rewards/autonomy_reward_fn/mean": 0.009999999776482582,
311
+ "rewards/autonomy_reward_fn/std": 0.0,
312
+ "step": 17,
313
+ "step_time": 5.571549678000338
314
+ },
315
+ {
316
+ "clip_ratio/high_max": 0.0,
317
+ "clip_ratio/high_mean": 0.0,
318
+ "clip_ratio/low_mean": 0.0,
319
+ "clip_ratio/low_min": 0.0,
320
+ "clip_ratio/region_mean": 0.0,
321
+ "entropy": 2.0457265377044678,
322
+ "epoch": 0.045,
323
+ "grad_norm": 0.0,
324
+ "learning_rate": 6.600000000000001e-06,
325
+ "loss": 0.0,
326
+ "step": 18,
327
+ "step_time": 0.12112131499998213
328
+ },
329
+ {
330
+ "clip_ratio/high_max": 0.0,
331
+ "clip_ratio/high_mean": 0.0,
332
+ "clip_ratio/low_mean": 0.0,
333
+ "clip_ratio/low_min": 0.0,
334
+ "clip_ratio/region_mean": 0.0,
335
+ "entropy": 2.4426045417785645,
336
+ "epoch": 0.0475,
337
+ "grad_norm": 0.0,
338
+ "learning_rate": 6.4000000000000006e-06,
339
+ "loss": 0.0,
340
+ "step": 19,
341
+ "step_time": 0.12062360999971133
342
+ },
343
+ {
344
+ "clip_ratio/high_max": 0.0,
345
+ "clip_ratio/high_mean": 0.0,
346
+ "clip_ratio/low_mean": 0.0,
347
+ "clip_ratio/low_min": 0.0,
348
+ "clip_ratio/region_mean": 0.0,
349
+ "entropy": 2.8799967765808105,
350
+ "epoch": 0.05,
351
+ "grad_norm": 0.0,
352
+ "learning_rate": 6.200000000000001e-06,
353
+ "loss": 0.0,
354
+ "step": 20,
355
+ "step_time": 0.1196468329999334
356
+ },
357
+ {
358
+ "clip_ratio/high_max": 0.0,
359
+ "clip_ratio/high_mean": 0.0,
360
+ "clip_ratio/low_mean": 0.0,
361
+ "clip_ratio/low_min": 0.0,
362
+ "clip_ratio/region_mean": 0.0,
363
+ "completions/clipped_ratio": 1.0,
364
+ "completions/max_length": 256.0,
365
+ "completions/max_terminated_length": 0.0,
366
+ "completions/mean_length": 256.0,
367
+ "completions/mean_terminated_length": 0.0,
368
+ "completions/min_length": 256.0,
369
+ "completions/min_terminated_length": 0.0,
370
+ "entropy": 2.4007647037506104,
371
+ "epoch": 0.0525,
372
+ "frac_reward_zero_std": 1.0,
373
+ "grad_norm": 0.0,
374
+ "learning_rate": 6e-06,
375
+ "loss": 0.0,
376
+ "num_tokens": 6756.0,
377
+ "reward": 0.009999999776482582,
378
+ "reward_std": 0.0,
379
+ "rewards/autonomy_reward_fn/mean": 0.009999999776482582,
380
+ "rewards/autonomy_reward_fn/std": 0.0,
381
+ "step": 21,
382
+ "step_time": 5.46478837199993
383
+ },
384
+ {
385
+ "clip_ratio/high_max": 0.0,
386
+ "clip_ratio/high_mean": 0.0,
387
+ "clip_ratio/low_mean": 0.0,
388
+ "clip_ratio/low_min": 0.0,
389
+ "clip_ratio/region_mean": 0.0,
390
+ "entropy": 2.0130887031555176,
391
+ "epoch": 0.055,
392
+ "grad_norm": 0.0,
393
+ "learning_rate": 5.8e-06,
394
+ "loss": 0.0,
395
+ "step": 22,
396
+ "step_time": 0.12104618500052311
397
+ },
398
+ {
399
+ "clip_ratio/high_max": 0.0,
400
+ "clip_ratio/high_mean": 0.0,
401
+ "clip_ratio/low_mean": 0.0,
402
+ "clip_ratio/low_min": 0.0,
403
+ "clip_ratio/region_mean": 0.0,
404
+ "entropy": 3.6849136352539062,
405
+ "epoch": 0.0575,
406
+ "grad_norm": 0.0,
407
+ "learning_rate": 5.600000000000001e-06,
408
+ "loss": 0.0,
409
+ "step": 23,
410
+ "step_time": 0.12017933200058906
411
+ },
412
+ {
413
+ "clip_ratio/high_max": 0.0,
414
+ "clip_ratio/high_mean": 0.0,
415
+ "clip_ratio/low_mean": 0.0,
416
+ "clip_ratio/low_min": 0.0,
417
+ "clip_ratio/region_mean": 0.0,
418
+ "entropy": 2.6699423789978027,
419
+ "epoch": 0.06,
420
+ "grad_norm": 0.0,
421
+ "learning_rate": 5.400000000000001e-06,
422
+ "loss": 0.0,
423
+ "step": 24,
424
+ "step_time": 0.11987412399957975
425
+ },
426
+ {
427
+ "clip_ratio/high_max": 0.0,
428
+ "clip_ratio/high_mean": 0.0,
429
+ "clip_ratio/low_mean": 0.0,
430
+ "clip_ratio/low_min": 0.0,
431
+ "clip_ratio/region_mean": 0.0,
432
+ "completions/clipped_ratio": 1.0,
433
+ "completions/max_length": 256.0,
434
+ "completions/max_terminated_length": 0.0,
435
+ "completions/mean_length": 256.0,
436
+ "completions/mean_terminated_length": 0.0,
437
+ "completions/min_length": 256.0,
438
+ "completions/min_terminated_length": 0.0,
439
+ "entropy": 2.353423595428467,
440
+ "epoch": 0.0625,
441
+ "frac_reward_zero_std": 1.0,
442
+ "grad_norm": 0.0,
443
+ "learning_rate": 5.2e-06,
444
+ "loss": 0.0,
445
+ "num_tokens": 7864.0,
446
+ "reward": 0.009999999776482582,
447
+ "reward_std": 0.0,
448
+ "rewards/autonomy_reward_fn/mean": 0.009999999776482582,
449
+ "rewards/autonomy_reward_fn/std": 0.0,
450
+ "step": 25,
451
+ "step_time": 5.475499750999916
452
+ }
453
+ ],
454
+ "logging_steps": 1,
455
+ "max_steps": 50,
456
+ "num_input_tokens_seen": 7864,
457
+ "num_train_epochs": 1,
458
+ "save_steps": 25,
459
+ "stateful_callbacks": {
460
+ "TrainerControl": {
461
+ "args": {
462
+ "should_epoch_stop": false,
463
+ "should_evaluate": false,
464
+ "should_log": false,
465
+ "should_save": true,
466
+ "should_training_stop": false
467
+ },
468
+ "attributes": {}
469
+ }
470
+ },
471
+ "total_flos": 0.0,
472
+ "train_batch_size": 1,
473
+ "trial_name": null,
474
+ "trial_params": null
475
+ }
checkpoint-25/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2d256cc6975c14fe69a6b3227efd5b2a62eb8616002cfbc058782540abc422f7
3
+ size 7185
checkpoint-50/chat_template.jinja ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {%- if tools %}
2
+ {{- '<|im_start|>system\n' }}
3
+ {%- if messages[0]['role'] == 'system' %}
4
+ {{- messages[0]['content'] }}
5
+ {%- else %}
6
+ {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
7
+ {%- endif %}
8
+ {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
9
+ {%- for tool in tools %}
10
+ {{- "\n" }}
11
+ {{- tool | tojson }}
12
+ {%- endfor %}
13
+ {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
14
+ {%- else %}
15
+ {%- if messages[0]['role'] == 'system' %}
16
+ {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
17
+ {%- else %}
18
+ {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
19
+ {%- endif %}
20
+ {%- endif %}
21
+ {%- for message in messages %}
22
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
23
+ {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
24
+ {%- elif message.role == "assistant" %}
25
+ {{- '<|im_start|>' + message.role }}
26
+ {%- if message.content %}
27
+ {{- '\n' + message.content }}
28
+ {%- endif %}
29
+ {%- for tool_call in message.tool_calls %}
30
+ {%- if tool_call.function is defined %}
31
+ {%- set tool_call = tool_call.function %}
32
+ {%- endif %}
33
+ {{- '\n<tool_call>\n{"name": "' }}
34
+ {{- tool_call.name }}
35
+ {{- '", "arguments": ' }}
36
+ {{- tool_call.arguments | tojson }}
37
+ {{- '}\n</tool_call>' }}
38
+ {%- endfor %}
39
+ {{- '<|im_end|>\n' }}
40
+ {%- elif message.role == "tool" %}
41
+ {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
42
+ {{- '<|im_start|>user' }}
43
+ {%- endif %}
44
+ {{- '\n<tool_response>\n' }}
45
+ {{- message.content }}
46
+ {{- '\n</tool_response>' }}
47
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
48
+ {{- '<|im_end|>\n' }}
49
+ {%- endif %}
50
+ {%- endif %}
51
+ {%- endfor %}
52
+ {%- if add_generation_prompt %}
53
+ {{- '<|im_start|>assistant\n' }}
54
+ {%- endif %}
checkpoint-50/config.json ADDED
@@ -0,0 +1,57 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen2ForCausalLM"
4
+ ],
5
+ "attention_dropout": 0.0,
6
+ "bos_token_id": null,
7
+ "dtype": "bfloat16",
8
+ "eos_token_id": 151645,
9
+ "hidden_act": "silu",
10
+ "hidden_size": 896,
11
+ "initializer_range": 0.02,
12
+ "intermediate_size": 4864,
13
+ "layer_types": [
14
+ "full_attention",
15
+ "full_attention",
16
+ "full_attention",
17
+ "full_attention",
18
+ "full_attention",
19
+ "full_attention",
20
+ "full_attention",
21
+ "full_attention",
22
+ "full_attention",
23
+ "full_attention",
24
+ "full_attention",
25
+ "full_attention",
26
+ "full_attention",
27
+ "full_attention",
28
+ "full_attention",
29
+ "full_attention",
30
+ "full_attention",
31
+ "full_attention",
32
+ "full_attention",
33
+ "full_attention",
34
+ "full_attention",
35
+ "full_attention",
36
+ "full_attention",
37
+ "full_attention"
38
+ ],
39
+ "max_position_embeddings": 32768,
40
+ "max_window_layers": 21,
41
+ "model_type": "qwen2",
42
+ "num_attention_heads": 14,
43
+ "num_hidden_layers": 24,
44
+ "num_key_value_heads": 2,
45
+ "pad_token_id": 151643,
46
+ "rms_norm_eps": 1e-06,
47
+ "rope_parameters": {
48
+ "rope_theta": 1000000.0,
49
+ "rope_type": "default"
50
+ },
51
+ "sliding_window": null,
52
+ "tie_word_embeddings": true,
53
+ "transformers_version": "5.6.2",
54
+ "use_cache": false,
55
+ "use_sliding_window": false,
56
+ "vocab_size": 151936
57
+ }
checkpoint-50/generation_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "do_sample": true,
3
+ "eos_token_id": [
4
+ 151645,
5
+ 151643
6
+ ],
7
+ "pad_token_id": 151643,
8
+ "repetition_penalty": 1.1,
9
+ "temperature": 0.7,
10
+ "top_k": 20,
11
+ "top_p": 0.8,
12
+ "transformers_version": "5.6.2"
13
+ }
checkpoint-50/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fdf756fa7fcbe7404d5c60e26bff1a0c8b8aa1f72ced49e7dd0210fe288fb7fe
3
+ size 988097824
checkpoint-50/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:055b61eef78c73abcca4dbfdccb61a4163db9c61516f25f8ad7f58e671e85828
3
+ size 1976378699
checkpoint-50/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c54cfd95c6cc3e287b93c5ac9469d6ee4290acf8ad1d1ae9c3bc8de0a3082b4a
3
+ size 14645
checkpoint-50/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:976f2bafa7669d9a7187fa276b6a26ad9abd7bff1427e837484e3c6b2ab4eff4
3
+ size 1465
checkpoint-50/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
3
+ size 11421892
checkpoint-50/tokenizer_config.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "backend": "tokenizers",
4
+ "bos_token": null,
5
+ "clean_up_tokenization_spaces": false,
6
+ "eos_token": "<|im_end|>",
7
+ "errors": "replace",
8
+ "extra_special_tokens": [
9
+ "<|im_start|>",
10
+ "<|im_end|>",
11
+ "<|object_ref_start|>",
12
+ "<|object_ref_end|>",
13
+ "<|box_start|>",
14
+ "<|box_end|>",
15
+ "<|quad_start|>",
16
+ "<|quad_end|>",
17
+ "<|vision_start|>",
18
+ "<|vision_end|>",
19
+ "<|vision_pad|>",
20
+ "<|image_pad|>",
21
+ "<|video_pad|>"
22
+ ],
23
+ "is_local": false,
24
+ "local_files_only": false,
25
+ "model_max_length": 131072,
26
+ "pad_token": "<|endoftext|>",
27
+ "padding_side": "left",
28
+ "split_special_tokens": false,
29
+ "tokenizer_class": "Qwen2Tokenizer",
30
+ "truncation_side": "left",
31
+ "unk_token": null
32
+ }
checkpoint-50/trainer_state.json ADDED
@@ -0,0 +1,903 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 0.125,
6
+ "eval_steps": 500,
7
+ "global_step": 50,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "clip_ratio/high_max": 0.0,
14
+ "clip_ratio/high_mean": 0.0,
15
+ "clip_ratio/low_mean": 0.0,
16
+ "clip_ratio/low_min": 0.0,
17
+ "clip_ratio/region_mean": 0.0,
18
+ "completions/clipped_ratio": 1.0,
19
+ "completions/max_length": 256.0,
20
+ "completions/max_terminated_length": 0.0,
21
+ "completions/mean_length": 256.0,
22
+ "completions/mean_terminated_length": 0.0,
23
+ "completions/min_length": 256.0,
24
+ "completions/min_terminated_length": 0.0,
25
+ "entropy": 1.975826621055603,
26
+ "epoch": 0.0025,
27
+ "frac_reward_zero_std": 1.0,
28
+ "grad_norm": 0.0,
29
+ "learning_rate": 1e-05,
30
+ "loss": 0.0,
31
+ "num_tokens": 1132.0,
32
+ "reward": 0.009999999776482582,
33
+ "reward_std": 0.0,
34
+ "rewards/autonomy_reward_fn/mean": 0.009999999776482582,
35
+ "rewards/autonomy_reward_fn/std": 0.0,
36
+ "step": 1,
37
+ "step_time": 5.940139313000145
38
+ },
39
+ {
40
+ "clip_ratio/high_max": 0.0,
41
+ "clip_ratio/high_mean": 0.0,
42
+ "clip_ratio/low_mean": 0.0,
43
+ "clip_ratio/low_min": 0.0,
44
+ "clip_ratio/region_mean": 0.0,
45
+ "entropy": 2.439243793487549,
46
+ "epoch": 0.005,
47
+ "grad_norm": 0.0,
48
+ "learning_rate": 9.800000000000001e-06,
49
+ "loss": 0.0,
50
+ "step": 2,
51
+ "step_time": 0.13269777900040935
52
+ },
53
+ {
54
+ "clip_ratio/high_max": 0.0,
55
+ "clip_ratio/high_mean": 0.0,
56
+ "clip_ratio/low_mean": 0.0,
57
+ "clip_ratio/low_min": 0.0,
58
+ "clip_ratio/region_mean": 0.0,
59
+ "entropy": 2.167536497116089,
60
+ "epoch": 0.0075,
61
+ "grad_norm": 0.0,
62
+ "learning_rate": 9.600000000000001e-06,
63
+ "loss": 0.0,
64
+ "step": 3,
65
+ "step_time": 0.12384193100024277
66
+ },
67
+ {
68
+ "clip_ratio/high_max": 0.0,
69
+ "clip_ratio/high_mean": 0.0,
70
+ "clip_ratio/low_mean": 0.0,
71
+ "clip_ratio/low_min": 0.0,
72
+ "clip_ratio/region_mean": 0.0,
73
+ "entropy": 2.270764112472534,
74
+ "epoch": 0.01,
75
+ "grad_norm": 0.0,
76
+ "learning_rate": 9.4e-06,
77
+ "loss": 0.0,
78
+ "step": 4,
79
+ "step_time": 0.12020948200006387
80
+ },
81
+ {
82
+ "clip_ratio/high_max": 0.0,
83
+ "clip_ratio/high_mean": 0.0,
84
+ "clip_ratio/low_mean": 0.0,
85
+ "clip_ratio/low_min": 0.0,
86
+ "clip_ratio/region_mean": 0.0,
87
+ "completions/clipped_ratio": 1.0,
88
+ "completions/max_length": 256.0,
89
+ "completions/max_terminated_length": 0.0,
90
+ "completions/mean_length": 256.0,
91
+ "completions/mean_terminated_length": 0.0,
92
+ "completions/min_length": 256.0,
93
+ "completions/min_terminated_length": 0.0,
94
+ "entropy": 1.7375574111938477,
95
+ "epoch": 0.0125,
96
+ "frac_reward_zero_std": 1.0,
97
+ "grad_norm": 0.0,
98
+ "learning_rate": 9.200000000000002e-06,
99
+ "loss": 0.0,
100
+ "num_tokens": 2240.0,
101
+ "reward": 0.009999999776482582,
102
+ "reward_std": 0.0,
103
+ "rewards/autonomy_reward_fn/mean": 0.009999999776482582,
104
+ "rewards/autonomy_reward_fn/std": 0.0,
105
+ "step": 5,
106
+ "step_time": 5.4328740460005065
107
+ },
108
+ {
109
+ "clip_ratio/high_max": 0.0,
110
+ "clip_ratio/high_mean": 0.0,
111
+ "clip_ratio/low_mean": 0.0,
112
+ "clip_ratio/low_min": 0.0,
113
+ "clip_ratio/region_mean": 0.0,
114
+ "entropy": 1.2307178974151611,
115
+ "epoch": 0.015,
116
+ "grad_norm": 0.0,
117
+ "learning_rate": 9e-06,
118
+ "loss": 0.0,
119
+ "step": 6,
120
+ "step_time": 0.12100409000049694
121
+ },
122
+ {
123
+ "clip_ratio/high_max": 0.0,
124
+ "clip_ratio/high_mean": 0.0,
125
+ "clip_ratio/low_mean": 0.0,
126
+ "clip_ratio/low_min": 0.0,
127
+ "clip_ratio/region_mean": 0.0,
128
+ "entropy": 2.5965301990509033,
129
+ "epoch": 0.0175,
130
+ "grad_norm": 0.0,
131
+ "learning_rate": 8.8e-06,
132
+ "loss": 0.0,
133
+ "step": 7,
134
+ "step_time": 0.11992437099979725
135
+ },
136
+ {
137
+ "clip_ratio/high_max": 0.0,
138
+ "clip_ratio/high_mean": 0.0,
139
+ "clip_ratio/low_mean": 0.0,
140
+ "clip_ratio/low_min": 0.0,
141
+ "clip_ratio/region_mean": 0.0,
142
+ "entropy": 3.000967502593994,
143
+ "epoch": 0.02,
144
+ "grad_norm": 0.0,
145
+ "learning_rate": 8.6e-06,
146
+ "loss": 0.0,
147
+ "step": 8,
148
+ "step_time": 0.12026366800000687
149
+ },
150
+ {
151
+ "clip_ratio/high_max": 0.0,
152
+ "clip_ratio/high_mean": 0.0,
153
+ "clip_ratio/low_mean": 0.0,
154
+ "clip_ratio/low_min": 0.0,
155
+ "clip_ratio/region_mean": 0.0,
156
+ "completions/clipped_ratio": 1.0,
157
+ "completions/max_length": 256.0,
158
+ "completions/max_terminated_length": 0.0,
159
+ "completions/mean_length": 256.0,
160
+ "completions/mean_terminated_length": 0.0,
161
+ "completions/min_length": 256.0,
162
+ "completions/min_terminated_length": 0.0,
163
+ "entropy": 2.1747303009033203,
164
+ "epoch": 0.0225,
165
+ "frac_reward_zero_std": 1.0,
166
+ "grad_norm": 0.0,
167
+ "learning_rate": 8.400000000000001e-06,
168
+ "loss": 0.0,
169
+ "num_tokens": 3372.0,
170
+ "reward": 0.009999999776482582,
171
+ "reward_std": 0.0,
172
+ "rewards/autonomy_reward_fn/mean": 0.009999999776482582,
173
+ "rewards/autonomy_reward_fn/std": 0.0,
174
+ "step": 9,
175
+ "step_time": 5.488657728999897
176
+ },
177
+ {
178
+ "clip_ratio/high_max": 0.0,
179
+ "clip_ratio/high_mean": 0.0,
180
+ "clip_ratio/low_mean": 0.0,
181
+ "clip_ratio/low_min": 0.0,
182
+ "clip_ratio/region_mean": 0.0,
183
+ "entropy": 2.2987818717956543,
184
+ "epoch": 0.025,
185
+ "grad_norm": 0.0,
186
+ "learning_rate": 8.2e-06,
187
+ "loss": 0.0,
188
+ "step": 10,
189
+ "step_time": 0.12148796400015272
190
+ },
191
+ {
192
+ "clip_ratio/high_max": 0.0,
193
+ "clip_ratio/high_mean": 0.0,
194
+ "clip_ratio/low_mean": 0.0,
195
+ "clip_ratio/low_min": 0.0,
196
+ "clip_ratio/region_mean": 0.0,
197
+ "entropy": 2.549589157104492,
198
+ "epoch": 0.0275,
199
+ "grad_norm": 0.0,
200
+ "learning_rate": 8.000000000000001e-06,
201
+ "loss": 0.0,
202
+ "step": 11,
203
+ "step_time": 0.1202768709999873
204
+ },
205
+ {
206
+ "clip_ratio/high_max": 0.0,
207
+ "clip_ratio/high_mean": 0.0,
208
+ "clip_ratio/low_mean": 0.0,
209
+ "clip_ratio/low_min": 0.0,
210
+ "clip_ratio/region_mean": 0.0,
211
+ "entropy": 2.3748786449432373,
212
+ "epoch": 0.03,
213
+ "grad_norm": 0.0,
214
+ "learning_rate": 7.800000000000002e-06,
215
+ "loss": 0.0,
216
+ "step": 12,
217
+ "step_time": 0.12146498200036149
218
+ },
219
+ {
220
+ "clip_ratio/high_max": 0.0,
221
+ "clip_ratio/high_mean": 0.0,
222
+ "clip_ratio/low_mean": 0.0,
223
+ "clip_ratio/low_min": 0.0,
224
+ "clip_ratio/region_mean": 0.0,
225
+ "completions/clipped_ratio": 1.0,
226
+ "completions/max_length": 256.0,
227
+ "completions/max_terminated_length": 0.0,
228
+ "completions/mean_length": 256.0,
229
+ "completions/mean_terminated_length": 0.0,
230
+ "completions/min_length": 256.0,
231
+ "completions/min_terminated_length": 0.0,
232
+ "entropy": 2.562595844268799,
233
+ "epoch": 0.0325,
234
+ "frac_reward_zero_std": 1.0,
235
+ "grad_norm": 0.0,
236
+ "learning_rate": 7.600000000000001e-06,
237
+ "loss": 0.0,
238
+ "num_tokens": 4504.0,
239
+ "reward": 0.009999999776482582,
240
+ "reward_std": 0.0,
241
+ "rewards/autonomy_reward_fn/mean": 0.009999999776482582,
242
+ "rewards/autonomy_reward_fn/std": 0.0,
243
+ "step": 13,
244
+ "step_time": 5.529226956000457
245
+ },
246
+ {
247
+ "clip_ratio/high_max": 0.0,
248
+ "clip_ratio/high_mean": 0.0,
249
+ "clip_ratio/low_mean": 0.0,
250
+ "clip_ratio/low_min": 0.0,
251
+ "clip_ratio/region_mean": 0.0,
252
+ "entropy": 2.9105567932128906,
253
+ "epoch": 0.035,
254
+ "grad_norm": 0.0,
255
+ "learning_rate": 7.4e-06,
256
+ "loss": 0.0,
257
+ "step": 14,
258
+ "step_time": 0.12003547699987394
259
+ },
260
+ {
261
+ "clip_ratio/high_max": 0.0,
262
+ "clip_ratio/high_mean": 0.0,
263
+ "clip_ratio/low_mean": 0.0,
264
+ "clip_ratio/low_min": 0.0,
265
+ "clip_ratio/region_mean": 0.0,
266
+ "entropy": 2.360198497772217,
267
+ "epoch": 0.0375,
268
+ "grad_norm": 0.0,
269
+ "learning_rate": 7.2000000000000005e-06,
270
+ "loss": 0.0,
271
+ "step": 15,
272
+ "step_time": 0.12056965900046634
273
+ },
274
+ {
275
+ "clip_ratio/high_max": 0.0,
276
+ "clip_ratio/high_mean": 0.0,
277
+ "clip_ratio/low_mean": 0.0,
278
+ "clip_ratio/low_min": 0.0,
279
+ "clip_ratio/region_mean": 0.0,
280
+ "entropy": 1.9906266927719116,
281
+ "epoch": 0.04,
282
+ "grad_norm": 0.0,
283
+ "learning_rate": 7e-06,
284
+ "loss": 0.0,
285
+ "step": 16,
286
+ "step_time": 0.11991975299952173
287
+ },
288
+ {
289
+ "clip_ratio/high_max": 0.0,
290
+ "clip_ratio/high_mean": 0.0,
291
+ "clip_ratio/low_mean": 0.0,
292
+ "clip_ratio/low_min": 0.0,
293
+ "clip_ratio/region_mean": 0.0,
294
+ "completions/clipped_ratio": 1.0,
295
+ "completions/max_length": 256.0,
296
+ "completions/max_terminated_length": 0.0,
297
+ "completions/mean_length": 256.0,
298
+ "completions/mean_terminated_length": 0.0,
299
+ "completions/min_length": 256.0,
300
+ "completions/min_terminated_length": 0.0,
301
+ "entropy": 2.4847490787506104,
302
+ "epoch": 0.0425,
303
+ "frac_reward_zero_std": 1.0,
304
+ "grad_norm": 0.0,
305
+ "learning_rate": 6.800000000000001e-06,
306
+ "loss": 0.0,
307
+ "num_tokens": 5636.0,
308
+ "reward": 0.009999999776482582,
309
+ "reward_std": 0.0,
310
+ "rewards/autonomy_reward_fn/mean": 0.009999999776482582,
311
+ "rewards/autonomy_reward_fn/std": 0.0,
312
+ "step": 17,
313
+ "step_time": 5.571549678000338
314
+ },
315
+ {
316
+ "clip_ratio/high_max": 0.0,
317
+ "clip_ratio/high_mean": 0.0,
318
+ "clip_ratio/low_mean": 0.0,
319
+ "clip_ratio/low_min": 0.0,
320
+ "clip_ratio/region_mean": 0.0,
321
+ "entropy": 2.0457265377044678,
322
+ "epoch": 0.045,
323
+ "grad_norm": 0.0,
324
+ "learning_rate": 6.600000000000001e-06,
325
+ "loss": 0.0,
326
+ "step": 18,
327
+ "step_time": 0.12112131499998213
328
+ },
329
+ {
330
+ "clip_ratio/high_max": 0.0,
331
+ "clip_ratio/high_mean": 0.0,
332
+ "clip_ratio/low_mean": 0.0,
333
+ "clip_ratio/low_min": 0.0,
334
+ "clip_ratio/region_mean": 0.0,
335
+ "entropy": 2.4426045417785645,
336
+ "epoch": 0.0475,
337
+ "grad_norm": 0.0,
338
+ "learning_rate": 6.4000000000000006e-06,
339
+ "loss": 0.0,
340
+ "step": 19,
341
+ "step_time": 0.12062360999971133
342
+ },
343
+ {
344
+ "clip_ratio/high_max": 0.0,
345
+ "clip_ratio/high_mean": 0.0,
346
+ "clip_ratio/low_mean": 0.0,
347
+ "clip_ratio/low_min": 0.0,
348
+ "clip_ratio/region_mean": 0.0,
349
+ "entropy": 2.8799967765808105,
350
+ "epoch": 0.05,
351
+ "grad_norm": 0.0,
352
+ "learning_rate": 6.200000000000001e-06,
353
+ "loss": 0.0,
354
+ "step": 20,
355
+ "step_time": 0.1196468329999334
356
+ },
357
+ {
358
+ "clip_ratio/high_max": 0.0,
359
+ "clip_ratio/high_mean": 0.0,
360
+ "clip_ratio/low_mean": 0.0,
361
+ "clip_ratio/low_min": 0.0,
362
+ "clip_ratio/region_mean": 0.0,
363
+ "completions/clipped_ratio": 1.0,
364
+ "completions/max_length": 256.0,
365
+ "completions/max_terminated_length": 0.0,
366
+ "completions/mean_length": 256.0,
367
+ "completions/mean_terminated_length": 0.0,
368
+ "completions/min_length": 256.0,
369
+ "completions/min_terminated_length": 0.0,
370
+ "entropy": 2.4007647037506104,
371
+ "epoch": 0.0525,
372
+ "frac_reward_zero_std": 1.0,
373
+ "grad_norm": 0.0,
374
+ "learning_rate": 6e-06,
375
+ "loss": 0.0,
376
+ "num_tokens": 6756.0,
377
+ "reward": 0.009999999776482582,
378
+ "reward_std": 0.0,
379
+ "rewards/autonomy_reward_fn/mean": 0.009999999776482582,
380
+ "rewards/autonomy_reward_fn/std": 0.0,
381
+ "step": 21,
382
+ "step_time": 5.46478837199993
383
+ },
384
+ {
385
+ "clip_ratio/high_max": 0.0,
386
+ "clip_ratio/high_mean": 0.0,
387
+ "clip_ratio/low_mean": 0.0,
388
+ "clip_ratio/low_min": 0.0,
389
+ "clip_ratio/region_mean": 0.0,
390
+ "entropy": 2.0130887031555176,
391
+ "epoch": 0.055,
392
+ "grad_norm": 0.0,
393
+ "learning_rate": 5.8e-06,
394
+ "loss": 0.0,
395
+ "step": 22,
396
+ "step_time": 0.12104618500052311
397
+ },
398
+ {
399
+ "clip_ratio/high_max": 0.0,
400
+ "clip_ratio/high_mean": 0.0,
401
+ "clip_ratio/low_mean": 0.0,
402
+ "clip_ratio/low_min": 0.0,
403
+ "clip_ratio/region_mean": 0.0,
404
+ "entropy": 3.6849136352539062,
405
+ "epoch": 0.0575,
406
+ "grad_norm": 0.0,
407
+ "learning_rate": 5.600000000000001e-06,
408
+ "loss": 0.0,
409
+ "step": 23,
410
+ "step_time": 0.12017933200058906
411
+ },
412
+ {
413
+ "clip_ratio/high_max": 0.0,
414
+ "clip_ratio/high_mean": 0.0,
415
+ "clip_ratio/low_mean": 0.0,
416
+ "clip_ratio/low_min": 0.0,
417
+ "clip_ratio/region_mean": 0.0,
418
+ "entropy": 2.6699423789978027,
419
+ "epoch": 0.06,
420
+ "grad_norm": 0.0,
421
+ "learning_rate": 5.400000000000001e-06,
422
+ "loss": 0.0,
423
+ "step": 24,
424
+ "step_time": 0.11987412399957975
425
+ },
426
+ {
427
+ "clip_ratio/high_max": 0.0,
428
+ "clip_ratio/high_mean": 0.0,
429
+ "clip_ratio/low_mean": 0.0,
430
+ "clip_ratio/low_min": 0.0,
431
+ "clip_ratio/region_mean": 0.0,
432
+ "completions/clipped_ratio": 1.0,
433
+ "completions/max_length": 256.0,
434
+ "completions/max_terminated_length": 0.0,
435
+ "completions/mean_length": 256.0,
436
+ "completions/mean_terminated_length": 0.0,
437
+ "completions/min_length": 256.0,
438
+ "completions/min_terminated_length": 0.0,
439
+ "entropy": 2.353423595428467,
440
+ "epoch": 0.0625,
441
+ "frac_reward_zero_std": 1.0,
442
+ "grad_norm": 0.0,
443
+ "learning_rate": 5.2e-06,
444
+ "loss": 0.0,
445
+ "num_tokens": 7864.0,
446
+ "reward": 0.009999999776482582,
447
+ "reward_std": 0.0,
448
+ "rewards/autonomy_reward_fn/mean": 0.009999999776482582,
449
+ "rewards/autonomy_reward_fn/std": 0.0,
450
+ "step": 25,
451
+ "step_time": 5.475499750999916
452
+ },
453
+ {
454
+ "clip_ratio/high_max": 0.0,
455
+ "clip_ratio/high_mean": 0.0,
456
+ "clip_ratio/low_mean": 0.0,
457
+ "clip_ratio/low_min": 0.0,
458
+ "clip_ratio/region_mean": 0.0,
459
+ "entropy": 1.6273622512817383,
460
+ "epoch": 0.065,
461
+ "grad_norm": 0.0,
462
+ "learning_rate": 5e-06,
463
+ "loss": 0.0,
464
+ "step": 26,
465
+ "step_time": 0.12144228599936469
466
+ },
467
+ {
468
+ "clip_ratio/high_max": 0.0,
469
+ "clip_ratio/high_mean": 0.0,
470
+ "clip_ratio/low_mean": 0.0,
471
+ "clip_ratio/low_min": 0.0,
472
+ "clip_ratio/region_mean": 0.0,
473
+ "entropy": 1.7559497356414795,
474
+ "epoch": 0.0675,
475
+ "grad_norm": 0.0,
476
+ "learning_rate": 4.800000000000001e-06,
477
+ "loss": 0.0,
478
+ "step": 27,
479
+ "step_time": 0.12019816300016828
480
+ },
481
+ {
482
+ "clip_ratio/high_max": 0.0,
483
+ "clip_ratio/high_mean": 0.0,
484
+ "clip_ratio/low_mean": 0.0,
485
+ "clip_ratio/low_min": 0.0,
486
+ "clip_ratio/region_mean": 0.0,
487
+ "entropy": 3.4082813262939453,
488
+ "epoch": 0.07,
489
+ "grad_norm": 0.0,
490
+ "learning_rate": 4.600000000000001e-06,
491
+ "loss": 0.0,
492
+ "step": 28,
493
+ "step_time": 0.12150602700057789
494
+ },
495
+ {
496
+ "clip_ratio/high_max": 0.0,
497
+ "clip_ratio/high_mean": 0.0,
498
+ "clip_ratio/low_mean": 0.0,
499
+ "clip_ratio/low_min": 0.0,
500
+ "clip_ratio/region_mean": 0.0,
501
+ "completions/clipped_ratio": 1.0,
502
+ "completions/max_length": 256.0,
503
+ "completions/max_terminated_length": 0.0,
504
+ "completions/mean_length": 256.0,
505
+ "completions/mean_terminated_length": 0.0,
506
+ "completions/min_length": 256.0,
507
+ "completions/min_terminated_length": 0.0,
508
+ "entropy": 2.0362539291381836,
509
+ "epoch": 0.0725,
510
+ "frac_reward_zero_std": 1.0,
511
+ "grad_norm": 0.0,
512
+ "learning_rate": 4.4e-06,
513
+ "loss": 0.0,
514
+ "num_tokens": 8972.0,
515
+ "reward": 0.009999999776482582,
516
+ "reward_std": 0.0,
517
+ "rewards/autonomy_reward_fn/mean": 0.009999999776482582,
518
+ "rewards/autonomy_reward_fn/std": 0.0,
519
+ "step": 29,
520
+ "step_time": 5.578138597999896
521
+ },
522
+ {
523
+ "clip_ratio/high_max": 0.0,
524
+ "clip_ratio/high_mean": 0.0,
525
+ "clip_ratio/low_mean": 0.0,
526
+ "clip_ratio/low_min": 0.0,
527
+ "clip_ratio/region_mean": 0.0,
528
+ "entropy": 1.9473274946212769,
529
+ "epoch": 0.075,
530
+ "grad_norm": 0.0,
531
+ "learning_rate": 4.2000000000000004e-06,
532
+ "loss": 0.0,
533
+ "step": 30,
534
+ "step_time": 0.11901681000017561
535
+ },
536
+ {
537
+ "clip_ratio/high_max": 0.0,
538
+ "clip_ratio/high_mean": 0.0,
539
+ "clip_ratio/low_mean": 0.0,
540
+ "clip_ratio/low_min": 0.0,
541
+ "clip_ratio/region_mean": 0.0,
542
+ "entropy": 3.764465570449829,
543
+ "epoch": 0.0775,
544
+ "grad_norm": 0.0,
545
+ "learning_rate": 4.000000000000001e-06,
546
+ "loss": 0.0,
547
+ "step": 31,
548
+ "step_time": 0.11898399900019285
549
+ },
550
+ {
551
+ "clip_ratio/high_max": 0.0,
552
+ "clip_ratio/high_mean": 0.0,
553
+ "clip_ratio/low_mean": 0.0,
554
+ "clip_ratio/low_min": 0.0,
555
+ "clip_ratio/region_mean": 0.0,
556
+ "entropy": 1.9196826219558716,
557
+ "epoch": 0.08,
558
+ "grad_norm": 0.0,
559
+ "learning_rate": 3.8000000000000005e-06,
560
+ "loss": 0.0,
561
+ "step": 32,
562
+ "step_time": 0.12208616299994901
563
+ },
564
+ {
565
+ "clip_ratio/high_max": 0.0,
566
+ "clip_ratio/high_mean": 0.0,
567
+ "clip_ratio/low_mean": 0.0,
568
+ "clip_ratio/low_min": 0.0,
569
+ "clip_ratio/region_mean": 0.0,
570
+ "completions/clipped_ratio": 1.0,
571
+ "completions/max_length": 256.0,
572
+ "completions/max_terminated_length": 0.0,
573
+ "completions/mean_length": 256.0,
574
+ "completions/mean_terminated_length": 0.0,
575
+ "completions/min_length": 256.0,
576
+ "completions/min_terminated_length": 0.0,
577
+ "entropy": 2.1861605644226074,
578
+ "epoch": 0.0825,
579
+ "frac_reward_zero_std": 1.0,
580
+ "grad_norm": 0.0,
581
+ "learning_rate": 3.6000000000000003e-06,
582
+ "loss": 0.0,
583
+ "num_tokens": 10104.0,
584
+ "reward": 0.009999999776482582,
585
+ "reward_std": 0.0,
586
+ "rewards/autonomy_reward_fn/mean": 0.009999999776482582,
587
+ "rewards/autonomy_reward_fn/std": 0.0,
588
+ "step": 33,
589
+ "step_time": 5.526427269999658
590
+ },
591
+ {
592
+ "clip_ratio/high_max": 0.0,
593
+ "clip_ratio/high_mean": 0.0,
594
+ "clip_ratio/low_mean": 0.0,
595
+ "clip_ratio/low_min": 0.0,
596
+ "clip_ratio/region_mean": 0.0,
597
+ "entropy": 1.9254075288772583,
598
+ "epoch": 0.085,
599
+ "grad_norm": 0.0,
600
+ "learning_rate": 3.4000000000000005e-06,
601
+ "loss": 0.0,
602
+ "step": 34,
603
+ "step_time": 0.11869688299975678
604
+ },
605
+ {
606
+ "clip_ratio/high_max": 0.0,
607
+ "clip_ratio/high_mean": 0.0,
608
+ "clip_ratio/low_mean": 0.0,
609
+ "clip_ratio/low_min": 0.0,
610
+ "clip_ratio/region_mean": 0.0,
611
+ "entropy": 2.2586331367492676,
612
+ "epoch": 0.0875,
613
+ "grad_norm": 0.0,
614
+ "learning_rate": 3.2000000000000003e-06,
615
+ "loss": 0.0,
616
+ "step": 35,
617
+ "step_time": 0.11914809700010665
618
+ },
619
+ {
620
+ "clip_ratio/high_max": 0.0,
621
+ "clip_ratio/high_mean": 0.0,
622
+ "clip_ratio/low_mean": 0.0,
623
+ "clip_ratio/low_min": 0.0,
624
+ "clip_ratio/region_mean": 0.0,
625
+ "entropy": 2.4637088775634766,
626
+ "epoch": 0.09,
627
+ "grad_norm": 0.0,
628
+ "learning_rate": 3e-06,
629
+ "loss": 0.0,
630
+ "step": 36,
631
+ "step_time": 0.1263795000004393
632
+ },
633
+ {
634
+ "clip_ratio/high_max": 0.0,
635
+ "clip_ratio/high_mean": 0.0,
636
+ "clip_ratio/low_mean": 0.0,
637
+ "clip_ratio/low_min": 0.0,
638
+ "clip_ratio/region_mean": 0.0,
639
+ "completions/clipped_ratio": 1.0,
640
+ "completions/max_length": 256.0,
641
+ "completions/max_terminated_length": 0.0,
642
+ "completions/mean_length": 256.0,
643
+ "completions/mean_terminated_length": 0.0,
644
+ "completions/min_length": 256.0,
645
+ "completions/min_terminated_length": 0.0,
646
+ "entropy": 3.3892905712127686,
647
+ "epoch": 0.0925,
648
+ "frac_reward_zero_std": 1.0,
649
+ "grad_norm": 0.0,
650
+ "learning_rate": 2.8000000000000003e-06,
651
+ "loss": 0.0,
652
+ "num_tokens": 11212.0,
653
+ "reward": 0.009999999776482582,
654
+ "reward_std": 0.0,
655
+ "rewards/autonomy_reward_fn/mean": 0.009999999776482582,
656
+ "rewards/autonomy_reward_fn/std": 0.0,
657
+ "step": 37,
658
+ "step_time": 5.478073480000603
659
+ },
660
+ {
661
+ "clip_ratio/high_max": 0.0,
662
+ "clip_ratio/high_mean": 0.0,
663
+ "clip_ratio/low_mean": 0.0,
664
+ "clip_ratio/low_min": 0.0,
665
+ "clip_ratio/region_mean": 0.0,
666
+ "entropy": 3.1603095531463623,
667
+ "epoch": 0.095,
668
+ "grad_norm": 0.0,
669
+ "learning_rate": 2.6e-06,
670
+ "loss": 0.0,
671
+ "step": 38,
672
+ "step_time": 0.12008928099930927
673
+ },
674
+ {
675
+ "clip_ratio/high_max": 0.0,
676
+ "clip_ratio/high_mean": 0.0,
677
+ "clip_ratio/low_mean": 0.0,
678
+ "clip_ratio/low_min": 0.0,
679
+ "clip_ratio/region_mean": 0.0,
680
+ "entropy": 2.8118975162506104,
681
+ "epoch": 0.0975,
682
+ "grad_norm": 0.0,
683
+ "learning_rate": 2.4000000000000003e-06,
684
+ "loss": 0.0,
685
+ "step": 39,
686
+ "step_time": 0.11953222700049082
687
+ },
688
+ {
689
+ "clip_ratio/high_max": 0.0,
690
+ "clip_ratio/high_mean": 0.0,
691
+ "clip_ratio/low_mean": 0.0,
692
+ "clip_ratio/low_min": 0.0,
693
+ "clip_ratio/region_mean": 0.0,
694
+ "entropy": 1.9064228534698486,
695
+ "epoch": 0.1,
696
+ "grad_norm": 0.0,
697
+ "learning_rate": 2.2e-06,
698
+ "loss": 0.0,
699
+ "step": 40,
700
+ "step_time": 0.11825968199991621
701
+ },
702
+ {
703
+ "clip_ratio/high_max": 0.0,
704
+ "clip_ratio/high_mean": 0.0,
705
+ "clip_ratio/low_mean": 0.0,
706
+ "clip_ratio/low_min": 0.0,
707
+ "clip_ratio/region_mean": 0.0,
708
+ "completions/clipped_ratio": 1.0,
709
+ "completions/max_length": 256.0,
710
+ "completions/max_terminated_length": 0.0,
711
+ "completions/mean_length": 256.0,
712
+ "completions/mean_terminated_length": 0.0,
713
+ "completions/min_length": 256.0,
714
+ "completions/min_terminated_length": 0.0,
715
+ "entropy": 2.5093579292297363,
716
+ "epoch": 0.1025,
717
+ "frac_reward_zero_std": 1.0,
718
+ "grad_norm": 0.0,
719
+ "learning_rate": 2.0000000000000003e-06,
720
+ "loss": 0.0,
721
+ "num_tokens": 12320.0,
722
+ "reward": 0.009999999776482582,
723
+ "reward_std": 0.0,
724
+ "rewards/autonomy_reward_fn/mean": 0.009999999776482582,
725
+ "rewards/autonomy_reward_fn/std": 0.0,
726
+ "step": 41,
727
+ "step_time": 5.509690339999906
728
+ },
729
+ {
730
+ "clip_ratio/high_max": 0.0,
731
+ "clip_ratio/high_mean": 0.0,
732
+ "clip_ratio/low_mean": 0.0,
733
+ "clip_ratio/low_min": 0.0,
734
+ "clip_ratio/region_mean": 0.0,
735
+ "entropy": 2.3179931640625,
736
+ "epoch": 0.105,
737
+ "grad_norm": 0.0,
738
+ "learning_rate": 1.8000000000000001e-06,
739
+ "loss": 0.0,
740
+ "step": 42,
741
+ "step_time": 0.12237605300015275
742
+ },
743
+ {
744
+ "clip_ratio/high_max": 0.0,
745
+ "clip_ratio/high_mean": 0.0,
746
+ "clip_ratio/low_mean": 0.0,
747
+ "clip_ratio/low_min": 0.0,
748
+ "clip_ratio/region_mean": 0.0,
749
+ "entropy": 1.4812008142471313,
750
+ "epoch": 0.1075,
751
+ "grad_norm": 0.0,
752
+ "learning_rate": 1.6000000000000001e-06,
753
+ "loss": 0.0,
754
+ "step": 43,
755
+ "step_time": 0.12001185999997688
756
+ },
757
+ {
758
+ "clip_ratio/high_max": 0.0,
759
+ "clip_ratio/high_mean": 0.0,
760
+ "clip_ratio/low_mean": 0.0,
761
+ "clip_ratio/low_min": 0.0,
762
+ "clip_ratio/region_mean": 0.0,
763
+ "entropy": 4.90458869934082,
764
+ "epoch": 0.11,
765
+ "grad_norm": 0.0,
766
+ "learning_rate": 1.4000000000000001e-06,
767
+ "loss": 0.0,
768
+ "step": 44,
769
+ "step_time": 0.11975958299990452
770
+ },
771
+ {
772
+ "clip_ratio/high_max": 0.0,
773
+ "clip_ratio/high_mean": 0.0,
774
+ "clip_ratio/low_mean": 0.0,
775
+ "clip_ratio/low_min": 0.0,
776
+ "clip_ratio/region_mean": 0.0,
777
+ "completions/clipped_ratio": 1.0,
778
+ "completions/max_length": 256.0,
779
+ "completions/max_terminated_length": 0.0,
780
+ "completions/mean_length": 256.0,
781
+ "completions/mean_terminated_length": 0.0,
782
+ "completions/min_length": 256.0,
783
+ "completions/min_terminated_length": 0.0,
784
+ "entropy": 2.2333452701568604,
785
+ "epoch": 0.1125,
786
+ "frac_reward_zero_std": 1.0,
787
+ "grad_norm": 0.0,
788
+ "learning_rate": 1.2000000000000002e-06,
789
+ "loss": 0.0,
790
+ "num_tokens": 13452.0,
791
+ "reward": 0.009999999776482582,
792
+ "reward_std": 0.0,
793
+ "rewards/autonomy_reward_fn/mean": 0.009999999776482582,
794
+ "rewards/autonomy_reward_fn/std": 0.0,
795
+ "step": 45,
796
+ "step_time": 5.512979769000594
797
+ },
798
+ {
799
+ "clip_ratio/high_max": 0.0,
800
+ "clip_ratio/high_mean": 0.0,
801
+ "clip_ratio/low_mean": 0.0,
802
+ "clip_ratio/low_min": 0.0,
803
+ "clip_ratio/region_mean": 0.0,
804
+ "entropy": 1.8680591583251953,
805
+ "epoch": 0.115,
806
+ "grad_norm": 0.0,
807
+ "learning_rate": 1.0000000000000002e-06,
808
+ "loss": 0.0,
809
+ "step": 46,
810
+ "step_time": 0.12362773699987883
811
+ },
812
+ {
813
+ "clip_ratio/high_max": 0.0,
814
+ "clip_ratio/high_mean": 0.0,
815
+ "clip_ratio/low_mean": 0.0,
816
+ "clip_ratio/low_min": 0.0,
817
+ "clip_ratio/region_mean": 0.0,
818
+ "entropy": 3.232543706893921,
819
+ "epoch": 0.1175,
820
+ "grad_norm": 0.0,
821
+ "learning_rate": 8.000000000000001e-07,
822
+ "loss": 0.0,
823
+ "step": 47,
824
+ "step_time": 0.12175974799993128
825
+ },
826
+ {
827
+ "clip_ratio/high_max": 0.0,
828
+ "clip_ratio/high_mean": 0.0,
829
+ "clip_ratio/low_mean": 0.0,
830
+ "clip_ratio/low_min": 0.0,
831
+ "clip_ratio/region_mean": 0.0,
832
+ "entropy": 2.694847583770752,
833
+ "epoch": 0.12,
834
+ "grad_norm": 0.0,
835
+ "learning_rate": 6.000000000000001e-07,
836
+ "loss": 0.0,
837
+ "step": 48,
838
+ "step_time": 0.11931696000010561
839
+ },
840
+ {
841
+ "clip_ratio/high_max": 0.0,
842
+ "clip_ratio/high_mean": 0.0,
843
+ "clip_ratio/low_mean": 0.0,
844
+ "clip_ratio/low_min": 0.0,
845
+ "clip_ratio/region_mean": 0.0,
846
+ "completions/clipped_ratio": 1.0,
847
+ "completions/max_length": 256.0,
848
+ "completions/max_terminated_length": 0.0,
849
+ "completions/mean_length": 256.0,
850
+ "completions/mean_terminated_length": 0.0,
851
+ "completions/min_length": 256.0,
852
+ "completions/min_terminated_length": 0.0,
853
+ "entropy": 2.182255268096924,
854
+ "epoch": 0.1225,
855
+ "frac_reward_zero_std": 1.0,
856
+ "grad_norm": 0.0,
857
+ "learning_rate": 4.0000000000000003e-07,
858
+ "loss": 0.0,
859
+ "num_tokens": 14572.0,
860
+ "reward": 0.009999999776482582,
861
+ "reward_std": 0.0,
862
+ "rewards/autonomy_reward_fn/mean": 0.009999999776482582,
863
+ "rewards/autonomy_reward_fn/std": 0.0,
864
+ "step": 49,
865
+ "step_time": 5.534827849000067
866
+ },
867
+ {
868
+ "clip_ratio/high_max": 0.0,
869
+ "clip_ratio/high_mean": 0.0,
870
+ "clip_ratio/low_mean": 0.0,
871
+ "clip_ratio/low_min": 0.0,
872
+ "clip_ratio/region_mean": 0.0,
873
+ "entropy": 2.535165309906006,
874
+ "epoch": 0.125,
875
+ "grad_norm": 0.0,
876
+ "learning_rate": 2.0000000000000002e-07,
877
+ "loss": 0.0,
878
+ "step": 50,
879
+ "step_time": 0.12092450500040286
880
+ }
881
+ ],
882
+ "logging_steps": 1,
883
+ "max_steps": 50,
884
+ "num_input_tokens_seen": 14572,
885
+ "num_train_epochs": 1,
886
+ "save_steps": 25,
887
+ "stateful_callbacks": {
888
+ "TrainerControl": {
889
+ "args": {
890
+ "should_epoch_stop": false,
891
+ "should_evaluate": false,
892
+ "should_log": false,
893
+ "should_save": true,
894
+ "should_training_stop": true
895
+ },
896
+ "attributes": {}
897
+ }
898
+ },
899
+ "total_flos": 0.0,
900
+ "train_batch_size": 1,
901
+ "trial_name": null,
902
+ "trial_params": null
903
+ }
checkpoint-50/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2d256cc6975c14fe69a6b3227efd5b2a62eb8616002cfbc058782540abc422f7
3
+ size 7185