Hugging Face's logo

hongzhuyi
/
models_

TensorBoard
Safetensors
Model card Files
xet
Metrics Community
hongzhuyi commited on
Commit
ee84420
·
verified ·
1 Parent(s): c3ba7ec

Upload folder using huggingface_hub

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +2 -0
  2. log/20250912-01:01:18.log +1 -0
  3. log/20250912-01:05:40.log +271 -0
  4. log/20250912-01:06:27.log +0 -0
  5. log/20250912-01:35:35.log +0 -0
  6. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/args.json +382 -0
  7. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/added_tokens.json +24 -0
  8. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/args.json +382 -0
  9. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/chat_template.jinja +54 -0
  10. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/config.json +59 -0
  11. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/generation_config.json +14 -0
  12. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/latest +1 -0
  13. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/merges.txt +0 -0
  14. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/model-00001-of-00004.safetensors +3 -0
  15. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/model-00002-of-00004.safetensors +3 -0
  16. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/model-00003-of-00004.safetensors +3 -0
  17. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/model-00004-of-00004.safetensors +3 -0
  18. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/model.safetensors.index.json +347 -0
  19. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/special_tokens_map.json +31 -0
  20. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/tokenizer.json +3 -0
  21. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/tokenizer_config.json +207 -0
  22. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/trainer_state.json +1437 -0
  23. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/training_args.bin +3 -0
  24. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/vocab.json +0 -0
  25. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/zero_to_fp32.py +760 -0
  26. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/added_tokens.json +24 -0
  27. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/args.json +382 -0
  28. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/chat_template.jinja +54 -0
  29. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/config.json +59 -0
  30. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/generation_config.json +14 -0
  31. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/latest +1 -0
  32. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/merges.txt +0 -0
  33. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/model-00001-of-00004.safetensors +3 -0
  34. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/model-00002-of-00004.safetensors +3 -0
  35. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/model-00003-of-00004.safetensors +3 -0
  36. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/model-00004-of-00004.safetensors +3 -0
  37. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/model.safetensors.index.json +347 -0
  38. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/special_tokens_map.json +31 -0
  39. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/tokenizer.json +3 -0
  40. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/tokenizer_config.json +207 -0
  41. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/trainer_state.json +731 -0
  42. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/training_args.bin +3 -0
  43. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/vocab.json +0 -0
  44. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/zero_to_fp32.py +760 -0
  45. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/images/eval_loss.png +0 -0
  46. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/images/eval_runtime.png +0 -0
  47. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/images/eval_samples_per_second.png +0 -0
  48. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/images/eval_steps_per_second.png +0 -0
  49. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/images/eval_token_acc.png +0 -0
  50. qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/images/train_epoch.png +0 -0
.gitattributes CHANGED
@@ -64,3 +64,5 @@ qwen2.5-7b-2225q-2069q-1369q-newbs-old-click-xep-lr5e-6/checkpoint-636/tokenizer
64
  qwen2.5-7b-2225q-2069q-1369q-newbs-old-click-xep-lr5e-6/checkpoint-954/tokenizer.json filter=lfs diff=lfs merge=lfs -text
65
  qwen2.5-7b-2000q-2000q-2000q-rft-newbs-old-click-xep-lr1e-6/checkpoint-361/tokenizer.json filter=lfs diff=lfs merge=lfs -text
66
  qwen2.5-7b-2000q-2000q-2000q-rft-newbs-old-click-xep-lr1e-6/checkpoint-722/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 
 
 
64
  qwen2.5-7b-2225q-2069q-1369q-newbs-old-click-xep-lr5e-6/checkpoint-954/tokenizer.json filter=lfs diff=lfs merge=lfs -text
65
  qwen2.5-7b-2000q-2000q-2000q-rft-newbs-old-click-xep-lr1e-6/checkpoint-361/tokenizer.json filter=lfs diff=lfs merge=lfs -text
66
  qwen2.5-7b-2000q-2000q-2000q-rft-newbs-old-click-xep-lr1e-6/checkpoint-722/tokenizer.json filter=lfs diff=lfs merge=lfs -text
67
+ qwen2.5-7b-2225q-206[[:space:]]q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/tokenizer.json filter=lfs diff=lfs merge=lfs -text
68
+ qwen2.5-7b-2225q-206[[:space:]]q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/tokenizer.json filter=lfs diff=lfs merge=lfs -text
log/20250912-01:01:18.log ADDED
@@ -0,0 +1 @@
 
 
1
+ test.sh: line 28: swift: command not found
log/20250912-01:05:40.log ADDED
@@ -0,0 +1,271 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ run sh: `/data/miniforge/envs/ms-swift/bin/python3.10 -m torch.distributed.run --nproc_per_node 8 /group/40143/hongzhuyi/ms-swift/swift/cli/sft.py --torch_dtype bfloat16 --freeze_llm false --freeze_aligner false --model Qwen/Qwen2.5-7B-Instruct --train_type full --dataset ./corr_hotpot_2083q_context_swift.jsonl ./corr_hotpot_new1369q_context_swift.jsonl ./corr_nq_2225q_context_swift.jsonl --model_type qwen2_5 --dataset_num_proc 100 --dataloader_num_workers 48 --split_dataset_ratio 0.001 --warmup_ratio 0.05 --num_train_epochs 2 --per_device_train_batch_size 2 --learning_rate 1e-6 --gradient_accumulation_steps 4 --eval_steps 2000 --save_strategy epoch --logging_steps 1 --deepspeed zero3 --max_length 16240 --output_dir ./output`
2
+
3
+ *****************************************
4
+ Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
5
+ *****************************************
6
+ [INFO:swift] Successfully registered `/group/40143/hongzhuyi/ms-swift/swift/llm/dataset/data/dataset_info.json`.
7
+ [INFO:swift] rank: 0, local_rank: 0, world_size: 8, local_world_size: 8
8
+ Downloading Model from https://www.modelscope.cn to directory: /usr/local/app/.cache/modelscope/hub/models/Qwen/Qwen2.5-7B-Instruct
9
+
10
+
11
+
12
+
13
+
14
+
15
+
16
+
17
+
18
+
19
+
20
+
21
+
22
+
23
+
24
+
25
+
26
+
27
+
28
+
29
+
30
+
31
+
32
+
33
+
34
+
35
+
36
+
37
+
38
+
39
+
40
+
41
+
42
+
43
+
44
+
45
+
46
+
47
+
48
+
49
+
50
+
51
+
52
+
53
+
54
+
55
+
56
+
57
+
58
+
59
+
60
+
61
+
62
+
63
+
64
+
65
+
66
+
67
+
68
+
69
+
70
+
71
+
72
+
73
+
74
+
75
+
76
+
77
+
78
+
79
+
80
+
81
+
82
+
83
+
84
+
85
+
86
+
87
+
88
+
89
+
90
+
91
+
92
+
93
+
94
+
95
+
96
+
97
+
98
+
99
+
100
+
101
+
102
+
103
+
104
+
105
+
106
+
107
+
108
+
109
+
110
+
111
+
112
+
113
+
114
+
115
+
116
+
117
+
118
+
119
+
120
+
121
+
122
+
123
+
124
+
125
+
126
+
127
+
128
+
129
+
130
+
131
+
132
+
133
+
134
+
135
+
136
+
137
+
138
+
139
+
140
+
141
+
142
+
143
+
144
+
145
+
146
+
147
+
148
+
149
+
150
+
151
+
152
+
153
+
154
+
155
+
156
+
157
+
158
+
159
+
160
+
161
+
162
+
163
+
164
+
165
+
166
+
167
+
168
+
169
+
170
+
171
+
172
+
173
+
174
+
175
+
176
+
177
+
178
+
179
+
180
+
181
+
182
+
183
+
184
+
185
+
186
+
187
+
188
+
189
+
190
+
191
+
192
+
193
+
194
+
195
+ Traceback (most recent call last):
196
+ File "/data/miniforge/envs/ms-swift/lib/python3.10/site-packages/transformers/utils/versions.py", line 102, in require_version
197
+ got_ver = importlib.metadata.version(pkg)
198
+ File "/data/miniforge/envs/ms-swift/lib/python3.10/importlib/metadata/__init__.py", line 996, in version
199
+ return distribution(distribution_name).version
200
+ File "/data/miniforge/envs/ms-swift/lib/python3.10/importlib/metadata/__init__.py", line 969, in distribution
201
+ return Distribution.from_name(distribution_name)
202
+ File "/data/miniforge/envs/ms-swift/lib/python3.10/importlib/metadata/__init__.py", line 548, in from_name
203
+ raise PackageNotFoundError(name)
204
+ importlib.metadata.PackageNotFoundError: No package metadata was found for deepspeed
205
+
206
+ During handling of the above exception, another exception occurred:
207
+
208
+ Traceback (most recent call last):
209
+ File "/group/40143/hongzhuyi/ms-swift/swift/cli/sft.py", line 10, in <module>
210
+ sft_main()
211
+ File "/group/40143/hongzhuyi/ms-swift/swift/llm/train/sft.py", line 328, in sft_main
212
+ return SwiftSft(args).main()
213
+ File "/group/40143/hongzhuyi/ms-swift/swift/llm/train/sft.py", line 27, in __init__
214
+ super().__init__(args)
215
+ File "/group/40143/hongzhuyi/ms-swift/swift/llm/base.py", line 19, in __init__
216
+ self.args = self._parse_args(args)
217
+ File "/group/40143/hongzhuyi/ms-swift/swift/llm/base.py", line 31, in _parse_args
218
+ args, remaining_argv = parse_args(self.args_class, args)
219
+ File "/group/40143/hongzhuyi/ms-swift/swift/utils/utils.py", line 152, in parse_args
220
+ args, remaining_args = parser.parse_args_into_dataclasses(argv, return_remaining_strings=True)
221
+ File "/data/miniforge/envs/ms-swift/lib/python3.10/site-packages/transformers/hf_argparser.py", line 358, in parse_args_into_dataclasses
222
+ obj = dtype(**inputs)
223
+ File "<string>", line 323, in __init__
224
+ File "/group/40143/hongzhuyi/ms-swift/swift/llm/argument/train_args.py", line 187, in __post_init__
225
+ self._init_deepspeed()
226
+ File "/group/40143/hongzhuyi/ms-swift/swift/llm/argument/train_args.py", line 203, in _init_deepspeed
227
+ require_version('deepspeed')
228
+ File "/data/miniforge/envs/ms-swift/lib/python3.10/site-packages/transformers/utils/versions.py", line 104, in require_version
229
+ raise importlib.metadata.PackageNotFoundError(
230
+ importlib.metadata.PackageNotFoundError: No package metadata was found for The 'deepspeed' distribution was not found and is required by this application.
231
+ W0912 01:06:02.339000 119518 site-packages/torch/distributed/elastic/multiprocessing/api.py:900] Sending process 119589 closing signal SIGTERM
232
+ W0912 01:06:02.339000 119518 site-packages/torch/distributed/elastic/multiprocessing/api.py:900] Sending process 119590 closing signal SIGTERM
233
+ W0912 01:06:02.339000 119518 site-packages/torch/distributed/elastic/multiprocessing/api.py:900] Sending process 119591 closing signal SIGTERM
234
+ W0912 01:06:02.340000 119518 site-packages/torch/distributed/elastic/multiprocessing/api.py:900] Sending process 119592 closing signal SIGTERM
235
+ W0912 01:06:02.340000 119518 site-packages/torch/distributed/elastic/multiprocessing/api.py:900] Sending process 119593 closing signal SIGTERM
236
+ W0912 01:06:02.342000 119518 site-packages/torch/distributed/elastic/multiprocessing/api.py:900] Sending process 119594 closing signal SIGTERM
237
+ W0912 01:06:02.342000 119518 site-packages/torch/distributed/elastic/multiprocessing/api.py:900] Sending process 119595 closing signal SIGTERM
238
+ E0912 01:06:02.457000 119518 site-packages/torch/distributed/elastic/multiprocessing/api.py:874] failed (exitcode: 1) local_rank: 7 (pid: 119596) of binary: /data/miniforge/envs/ms-swift/bin/python3.10
239
+ Traceback (most recent call last):
240
+ File "/data/miniforge/envs/ms-swift/lib/python3.10/runpy.py", line 196, in _run_module_as_main
241
+ return _run_code(code, main_globals, None,
242
+ File "/data/miniforge/envs/ms-swift/lib/python3.10/runpy.py", line 86, in _run_code
243
+ exec(code, run_globals)
244
+ File "/data/miniforge/envs/ms-swift/lib/python3.10/site-packages/torch/distributed/run.py", line 905, in <module>
245
+ main()
246
+ File "/data/miniforge/envs/ms-swift/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper
247
+ return f(*args, **kwargs)
248
+ File "/data/miniforge/envs/ms-swift/lib/python3.10/site-packages/torch/distributed/run.py", line 901, in main
249
+ run(args)
250
+ File "/data/miniforge/envs/ms-swift/lib/python3.10/site-packages/torch/distributed/run.py", line 892, in run
251
+ elastic_launch(
252
+ File "/data/miniforge/envs/ms-swift/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 143, in __call__
253
+ return launch_agent(self._config, self._entrypoint, list(args))
254
+ File "/data/miniforge/envs/ms-swift/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 277, in launch_agent
255
+ raise ChildFailedError(
256
+ torch.distributed.elastic.multiprocessing.errors.ChildFailedError:
257
+ ============================================================
258
+ /group/40143/hongzhuyi/ms-swift/swift/cli/sft.py FAILED
259
+ ------------------------------------------------------------
260
+ Failures:
261
+ <NO_OTHER_FAILURES>
262
+ ------------------------------------------------------------
263
+ Root Cause (first observed failure):
264
+ [0]:
265
+ time : 2025-09-12_01:06:02
266
+ host : TENCENT64.site
267
+ rank : 7 (local_rank: 7)
268
+ exitcode : 1 (pid: 119596)
269
+ error_file: <N/A>
270
+ traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
271
+ ============================================================
log/20250912-01:06:27.log ADDED
The diff for this file is too large to render. See raw diff
 
log/20250912-01:35:35.log ADDED
The diff for this file is too large to render. See raw diff
 
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/args.json ADDED
@@ -0,0 +1,382 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "output_dir": "/group/40143/hongzhuyi/ms-swift/output/v1-20250912-013615",
3
+ "overwrite_output_dir": false,
4
+ "do_train": false,
5
+ "do_eval": false,
6
+ "do_predict": false,
7
+ "eval_strategy": "epoch",
8
+ "prediction_loss_only": false,
9
+ "per_device_train_batch_size": 2,
10
+ "per_device_eval_batch_size": 1,
11
+ "per_gpu_train_batch_size": null,
12
+ "per_gpu_eval_batch_size": null,
13
+ "gradient_accumulation_steps": 4,
14
+ "eval_accumulation_steps": null,
15
+ "eval_delay": 0,
16
+ "torch_empty_cache_steps": null,
17
+ "learning_rate": 1e-06,
18
+ "weight_decay": 0.1,
19
+ "adam_beta1": 0.9,
20
+ "adam_beta2": 0.95,
21
+ "adam_epsilon": 1e-08,
22
+ "max_grad_norm": 1.0,
23
+ "num_train_epochs": 2.0,
24
+ "max_steps": -1,
25
+ "lr_scheduler_type": "cosine",
26
+ "lr_scheduler_kwargs": null,
27
+ "warmup_ratio": 0.05,
28
+ "warmup_steps": 0,
29
+ "log_level": "passive",
30
+ "log_level_replica": "warning",
31
+ "log_on_each_node": true,
32
+ "logging_dir": "/group/40143/hongzhuyi/ms-swift/output/v1-20250912-013615/runs",
33
+ "logging_strategy": "steps",
34
+ "logging_first_step": true,
35
+ "logging_steps": 1,
36
+ "logging_nan_inf_filter": true,
37
+ "save_strategy": "epoch",
38
+ "save_steps": 500,
39
+ "save_total_limit": null,
40
+ "save_safetensors": true,
41
+ "save_on_each_node": false,
42
+ "save_only_model": false,
43
+ "restore_callback_states_from_checkpoint": false,
44
+ "no_cuda": false,
45
+ "use_cpu": false,
46
+ "use_mps_device": false,
47
+ "seed": 42,
48
+ "data_seed": 42,
49
+ "jit_mode_eval": false,
50
+ "use_ipex": false,
51
+ "bf16": true,
52
+ "fp16": false,
53
+ "fp16_opt_level": "O1",
54
+ "half_precision_backend": "auto",
55
+ "bf16_full_eval": false,
56
+ "fp16_full_eval": false,
57
+ "tf32": null,
58
+ "local_rank": 0,
59
+ "ddp_backend": null,
60
+ "tpu_num_cores": null,
61
+ "tpu_metrics_debug": false,
62
+ "debug": null,
63
+ "dataloader_drop_last": false,
64
+ "eval_steps": 2000.0,
65
+ "dataloader_num_workers": 48,
66
+ "dataloader_prefetch_factor": null,
67
+ "past_index": -1,
68
+ "run_name": "/group/40143/hongzhuyi/ms-swift/output/v1-20250912-013615",
69
+ "disable_tqdm": null,
70
+ "remove_unused_columns": true,
71
+ "label_names": null,
72
+ "load_best_model_at_end": false,
73
+ "metric_for_best_model": "loss",
74
+ "greater_is_better": false,
75
+ "ignore_data_skip": false,
76
+ "fsdp": "",
77
+ "fsdp_min_num_params": 0,
78
+ "fsdp_config": null,
79
+ "fsdp_transformer_layer_cls_to_wrap": null,
80
+ "accelerator_config": {
81
+ "dispatch_batches": false
82
+ },
83
+ "deepspeed": {
84
+ "fp16": {
85
+ "enabled": "auto",
86
+ "loss_scale": 0,
87
+ "loss_scale_window": 1000,
88
+ "initial_scale_power": 16,
89
+ "hysteresis": 2,
90
+ "min_loss_scale": 1
91
+ },
92
+ "bf16": {
93
+ "enabled": "auto"
94
+ },
95
+ "zero_optimization": {
96
+ "stage": 3,
97
+ "offload_optimizer": {
98
+ "device": "none",
99
+ "pin_memory": true
100
+ },
101
+ "offload_param": {
102
+ "device": "none",
103
+ "pin_memory": true
104
+ },
105
+ "overlap_comm": false,
106
+ "contiguous_gradients": true,
107
+ "sub_group_size": 1000000000.0,
108
+ "reduce_bucket_size": "auto",
109
+ "zero_quantized_weights": false,
110
+ "zero_quantized_gradients": false,
111
+ "stage3_prefetch_bucket_size": "auto",
112
+ "stage3_param_persistence_threshold": "auto",
113
+ "stage3_max_live_parameters": 1000000000.0,
114
+ "stage3_max_reuse_distance": 1000000000.0,
115
+ "stage3_gather_16bit_weights_on_model_save": true
116
+ },
117
+ "gradient_accumulation_steps": "auto",
118
+ "gradient_clipping": "auto",
119
+ "steps_per_print": 2000,
120
+ "train_batch_size": "auto",
121
+ "train_micro_batch_size_per_gpu": "auto",
122
+ "wall_clock_breakdown": false
123
+ },
124
+ "label_smoothing_factor": 0.0,
125
+ "optim": "adamw_torch_fused",
126
+ "optim_args": null,
127
+ "adafactor": false,
128
+ "group_by_length": false,
129
+ "length_column_name": "length",
130
+ "report_to": [
131
+ "tensorboard"
132
+ ],
133
+ "ddp_find_unused_parameters": null,
134
+ "ddp_bucket_cap_mb": null,
135
+ "ddp_broadcast_buffers": null,
136
+ "dataloader_pin_memory": true,
137
+ "dataloader_persistent_workers": false,
138
+ "skip_memory_metrics": true,
139
+ "use_legacy_prediction_loop": false,
140
+ "push_to_hub": false,
141
+ "resume_from_checkpoint": null,
142
+ "hub_model_id": null,
143
+ "hub_strategy": "every_save",
144
+ "hub_token": null,
145
+ "hub_private_repo": null,
146
+ "hub_always_push": false,
147
+ "hub_revision": null,
148
+ "gradient_checkpointing": true,
149
+ "gradient_checkpointing_kwargs": null,
150
+ "include_inputs_for_metrics": false,
151
+ "include_for_metrics": [],
152
+ "eval_do_concat_batches": true,
153
+ "fp16_backend": "auto",
154
+ "push_to_hub_model_id": null,
155
+ "push_to_hub_organization": null,
156
+ "push_to_hub_token": null,
157
+ "mp_parameters": "",
158
+ "auto_find_batch_size": false,
159
+ "full_determinism": false,
160
+ "torchdynamo": null,
161
+ "ray_scope": "last",
162
+ "ddp_timeout": 18000000,
163
+ "torch_compile": false,
164
+ "torch_compile_backend": null,
165
+ "torch_compile_mode": null,
166
+ "include_tokens_per_second": false,
167
+ "include_num_input_tokens_seen": false,
168
+ "neftune_noise_alpha": null,
169
+ "optim_target_modules": null,
170
+ "batch_eval_metrics": false,
171
+ "eval_on_start": false,
172
+ "use_liger_kernel": false,
173
+ "liger_kernel_config": null,
174
+ "eval_use_gather_object": false,
175
+ "average_tokens_across_devices": true,
176
+ "sortish_sampler": false,
177
+ "predict_with_generate": false,
178
+ "generation_max_length": null,
179
+ "generation_num_beams": null,
180
+ "generation_config": null,
181
+ "tuner_backend": "peft",
182
+ "vit_gradient_checkpointing": null,
183
+ "router_aux_loss_coef": 0.0,
184
+ "enable_dft_loss": false,
185
+ "check_model": true,
186
+ "acc_strategy": "token",
187
+ "train_dataloader_shuffle": true,
188
+ "max_epochs": null,
189
+ "aligner_lr": null,
190
+ "vit_lr": null,
191
+ "use_logits_to_keep": null,
192
+ "channels": null,
193
+ "ds3_gather_for_generation": true,
194
+ "resume_only_model": false,
195
+ "optimizer": null,
196
+ "loss_type": null,
197
+ "metric": null,
198
+ "eval_use_evalscope": false,
199
+ "eval_dataset": [],
200
+ "eval_dataset_args": null,
201
+ "eval_limit": null,
202
+ "eval_generation_config": null,
203
+ "extra_eval_args": null,
204
+ "use_flash_ckpt": false,
205
+ "model": "Qwen/Qwen2.5-7B-Instruct",
206
+ "model_type": "qwen2_5",
207
+ "model_revision": null,
208
+ "task_type": "causal_lm",
209
+ "torch_dtype": "bfloat16",
210
+ "attn_impl": null,
211
+ "new_special_tokens": [],
212
+ "num_labels": null,
213
+ "problem_type": null,
214
+ "rope_scaling": null,
215
+ "device_map": null,
216
+ "max_memory": {},
217
+ "max_model_len": null,
218
+ "local_repo_path": null,
219
+ "init_strategy": null,
220
+ "template": "qwen2_5",
221
+ "system": null,
222
+ "max_length": 16240,
223
+ "truncation_strategy": "delete",
224
+ "max_pixels": null,
225
+ "agent_template": null,
226
+ "norm_bbox": null,
227
+ "use_chat_template": true,
228
+ "padding_free": false,
229
+ "padding_side": "right",
230
+ "loss_scale": "default",
231
+ "sequence_parallel_size": 1,
232
+ "response_prefix": null,
233
+ "template_backend": "swift",
234
+ "dataset": [
235
+ "./corr_hotpot_2083q_context_swift.jsonl",
236
+ "./corr_hotpot_new1369q_context_swift.jsonl",
237
+ "./corr_nq_2225q_context_swift.jsonl"
238
+ ],
239
+ "val_dataset": [],
240
+ "split_dataset_ratio": 0.001,
241
+ "dataset_num_proc": 100,
242
+ "load_from_cache_file": true,
243
+ "dataset_shuffle": true,
244
+ "val_dataset_shuffle": false,
245
+ "streaming": false,
246
+ "interleave_prob": null,
247
+ "stopping_strategy": "first_exhausted",
248
+ "shuffle_buffer_size": 1000,
249
+ "download_mode": "reuse_dataset_if_exists",
250
+ "columns": {},
251
+ "strict": false,
252
+ "model_name": null,
253
+ "model_author": null,
254
+ "custom_dataset_info": [],
255
+ "quant_method": null,
256
+ "quant_bits": null,
257
+ "hqq_axis": null,
258
+ "bnb_4bit_compute_dtype": "bfloat16",
259
+ "bnb_4bit_quant_type": "nf4",
260
+ "bnb_4bit_use_double_quant": true,
261
+ "bnb_4bit_quant_storage": null,
262
+ "max_new_tokens": 64,
263
+ "temperature": 0.0,
264
+ "top_k": null,
265
+ "top_p": null,
266
+ "repetition_penalty": null,
267
+ "num_beams": 1,
268
+ "stream": false,
269
+ "stop_words": [],
270
+ "logprobs": false,
271
+ "top_logprobs": null,
272
+ "ckpt_dir": null,
273
+ "lora_modules": [],
274
+ "train_type": "full",
275
+ "adapters": [],
276
+ "external_plugins": [],
277
+ "model_kwargs": {},
278
+ "load_args": false,
279
+ "load_data_args": false,
280
+ "packing": false,
281
+ "packing_length": null,
282
+ "lazy_tokenize": false,
283
+ "cached_dataset": [],
284
+ "custom_register_path": [],
285
+ "use_hf": false,
286
+ "ignore_args_error": false,
287
+ "use_swift_lora": false,
288
+ "freeze_parameters": [],
289
+ "freeze_parameters_regex": null,
290
+ "freeze_parameters_ratio": 0.0,
291
+ "trainable_parameters": [],
292
+ "trainable_parameters_regex": null,
293
+ "freeze_llm": false,
294
+ "freeze_vit": true,
295
+ "freeze_aligner": false,
296
+ "target_modules": [
297
+ "all-linear"
298
+ ],
299
+ "target_regex": null,
300
+ "modules_to_save": [],
301
+ "lora_rank": 8,
302
+ "lora_alpha": 32,
303
+ "lora_dropout": 0.05,
304
+ "lora_bias": "none",
305
+ "lora_dtype": null,
306
+ "lorap_lr_ratio": null,
307
+ "use_rslora": false,
308
+ "use_dora": false,
309
+ "lora_ga_batch_size": 2,
310
+ "lora_ga_iters": 2,
311
+ "lora_ga_max_length": 1024,
312
+ "lora_ga_direction": "ArB2r",
313
+ "lora_ga_scale": "stable",
314
+ "lora_ga_stable_gamma": 16,
315
+ "init_weights": true,
316
+ "fourier_n_frequency": 2000,
317
+ "fourier_scaling": 300.0,
318
+ "boft_block_size": 4,
319
+ "boft_block_num": 0,
320
+ "boft_n_butterfly_factor": 1,
321
+ "boft_dropout": 0.0,
322
+ "vera_rank": 256,
323
+ "vera_projection_prng_key": 0,
324
+ "vera_dropout": 0.0,
325
+ "vera_d_initial": 0.1,
326
+ "adapter_act": "gelu",
327
+ "adapter_length": 128,
328
+ "use_galore": false,
329
+ "galore_target_modules": null,
330
+ "galore_rank": 128,
331
+ "galore_update_proj_gap": 50,
332
+ "galore_scale": 1.0,
333
+ "galore_proj_type": "std",
334
+ "galore_optim_per_parameter": false,
335
+ "galore_with_embedding": false,
336
+ "galore_quantization": false,
337
+ "galore_proj_quant": false,
338
+ "galore_proj_bits": 4,
339
+ "galore_proj_group_size": 256,
340
+ "galore_cos_threshold": 0.4,
341
+ "galore_gamma_proj": 2,
342
+ "galore_queue_size": 5,
343
+ "adalora_target_r": 8,
344
+ "adalora_init_r": 12,
345
+ "adalora_tinit": 0,
346
+ "adalora_tfinal": 0,
347
+ "adalora_deltaT": 1,
348
+ "adalora_beta1": 0.85,
349
+ "adalora_beta2": 0.85,
350
+ "adalora_orth_reg_weight": 0.5,
351
+ "llamapro_num_new_blocks": 4,
352
+ "llamapro_num_groups": null,
353
+ "lisa_activated_layers": 0,
354
+ "lisa_step_interval": 20,
355
+ "reft_layer_key": null,
356
+ "reft_layers": null,
357
+ "reft_rank": 4,
358
+ "reft_intervention_type": "LoreftIntervention",
359
+ "reft_args": null,
360
+ "swanlab_token": null,
361
+ "swanlab_project": null,
362
+ "swanlab_workspace": null,
363
+ "swanlab_exp_name": null,
364
+ "swanlab_lark_webhook_url": null,
365
+ "swanlab_lark_secret": null,
366
+ "swanlab_mode": "cloud",
367
+ "add_version": true,
368
+ "create_checkpoint_symlink": false,
369
+ "zero_hpz_partition_size": null,
370
+ "deepspeed_autotp_size": null,
371
+ "early_stop_interval": null,
372
+ "rank": 0,
373
+ "global_world_size": 8,
374
+ "local_world_size": 8,
375
+ "model_suffix": "Qwen2.5-7B-Instruct",
376
+ "model_info": "ModelInfo(model_type='qwen2_5', model_dir='/usr/local/app/.cache/modelscope/hub/models/Qwen/Qwen2___5-7B-Instruct', torch_dtype=torch.bfloat16, max_model_len=32768, quant_method=None, quant_bits=None, rope_scaling=None, is_moe_model=False, config=None, task_type='causal_lm', num_labels=None)",
377
+ "model_meta": "ModelMeta(model_type='qwen2_5', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct', hf_model_id='Qwen/Qwen2.5-3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct', hf_model_id='Qwen/Qwen2.5-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct', hf_model_id='Qwen/Qwen2.5-14B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct', hf_model_id='Qwen/Qwen2.5-32B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct', hf_model_id='Qwen/Qwen2.5-72B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B', hf_model_id='Qwen/Qwen2.5-0.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B', hf_model_id='Qwen/Qwen2.5-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B', hf_model_id='Qwen/Qwen2.5-3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B', hf_model_id='Qwen/Qwen2.5-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B', hf_model_id='Qwen/Qwen2.5-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B', hf_model_id='Qwen/Qwen2.5-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B', hf_model_id='Qwen/Qwen2.5-72B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-3B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-7B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-14B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-32B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-72B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B', hf_model_id='Qwen/Qwen2.5-Coder-0.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B', hf_model_id='Qwen/Qwen2.5-Coder-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B', hf_model_id='Qwen/Qwen2.5-Coder-3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B', hf_model_id='Qwen/Qwen2.5-Coder-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B', hf_model_id='Qwen/Qwen2.5-Coder-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B', hf_model_id='Qwen/Qwen2.5-Coder-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=['coding']), ModelGroup(models=[Model(ms_model_id='moonshotai/Kimi-Dev-72B', hf_model_id='moonshotai/Kimi-Dev-72B', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[])], template='qwen2_5', get_function=<function get_model_tokenizer_with_flash_attn at 0x7f4f5af767a0>, model_arch=ModelKeys(arch_name='llama', embedding='model.embed_tokens', module_list='model.layers', lm_head='lm_head', q_proj='model.layers.{}.self_attn.q_proj', k_proj='model.layers.{}.self_attn.k_proj', v_proj='model.layers.{}.self_attn.v_proj', o_proj='model.layers.{}.self_attn.o_proj', attention='model.layers.{}.self_attn', mlp='model.layers.{}.mlp', down_proj='model.layers.{}.mlp.down_proj', qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None), architectures=['Qwen2ForCausalLM'], additional_saved_files=[], torch_dtype=None, is_multimodal=False, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.37'], tags=[])",
378
+ "model_dir": "/usr/local/app/.cache/modelscope/hub/models/Qwen/Qwen2___5-7B-Instruct",
379
+ "hub": "<class 'swift.hub.hub.MSHub'>",
380
+ "evaluation_strategy": "epoch",
381
+ "training_args": "Seq2SeqTrainingArguments(output_dir='/group/40143/hongzhuyi/ms-swift/output/v1-20250912-013615', overwrite_output_dir=False, do_train=False, do_eval=True, do_predict=False, eval_strategy=<IntervalStrategy.EPOCH: 'epoch'>, prediction_loss_only=False, per_device_train_batch_size=2, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=4, eval_accumulation_steps=None, eval_delay=0, torch_empty_cache_steps=None, learning_rate=1e-06, weight_decay=0.1, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=2.0, max_steps=-1, lr_scheduler_type=<SchedulerType.COSINE: 'cosine'>, lr_scheduler_kwargs=None, warmup_ratio=0.05, warmup_steps=0, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='/group/40143/hongzhuyi/ms-swift/output/v1-20250912-013615/runs', logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_first_step=True, logging_steps=1, logging_nan_inf_filter=True, save_strategy=<SaveStrategy.EPOCH: 'epoch'>, save_steps=500, save_total_limit=None, save_safetensors=True, save_on_each_node=False, save_only_model=False, restore_callback_states_from_checkpoint=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=42, data_seed=42, jit_mode_eval=False, use_ipex=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=2000.0, dataloader_num_workers=48, dataloader_prefetch_factor=10, past_index=-1, run_name='/group/40143/hongzhuyi/ms-swift/output/v1-20250912-013615', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model='loss', greater_is_better=False, ignore_data_skip=False, fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), deepspeed={'fp16': {'enabled': 'auto', 'loss_scale': 0, 'loss_scale_window': 1000, 'initial_scale_power': 16, 'hysteresis': 2, 'min_loss_scale': 1}, 'bf16': {'enabled': 'auto'}, 'zero_optimization': {'stage': 3, 'offload_optimizer': {'device': 'none', 'pin_memory': True}, 'offload_param': {'device': 'none', 'pin_memory': True}, 'overlap_comm': False, 'contiguous_gradients': True, 'sub_group_size': 1000000000.0, 'reduce_bucket_size': 'auto', 'zero_quantized_weights': False, 'zero_quantized_gradients': False, 'stage3_prefetch_bucket_size': 'auto', 'stage3_param_persistence_threshold': 'auto', 'stage3_max_live_parameters': 1000000000.0, 'stage3_max_reuse_distance': 1000000000.0, 'stage3_gather_16bit_weights_on_model_save': True}, 'gradient_accumulation_steps': 'auto', 'gradient_clipping': 'auto', 'steps_per_print': 2000, 'train_batch_size': 'auto', 'train_micro_batch_size_per_gpu': 'auto', 'wall_clock_breakdown': False}, label_smoothing_factor=0.0, optim=<OptimizerNames.ADAMW_TORCH_FUSED: 'adamw_torch_fused'>, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], ddp_find_unused_parameters=None, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, dataloader_persistent_workers=False, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_token=None, hub_private_repo=None, hub_always_push=False, hub_revision=None, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, include_inputs_for_metrics=False, include_for_metrics=[], eval_do_concat_batches=True, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=18000000, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, include_tokens_per_second=None, include_num_input_tokens_seen=None, neftune_noise_alpha=None, optim_target_modules=None, batch_eval_metrics=False, eval_on_start=False, use_liger_kernel=False, liger_kernel_config=None, eval_use_gather_object=False, average_tokens_across_devices=None, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=None, tuner_backend='peft', vit_gradient_checkpointing=True, router_aux_loss_coef=0.0, enable_dft_loss=False, check_model=True, acc_strategy='token', train_dataloader_shuffle=True, max_epochs=None, aligner_lr=None, vit_lr=None, use_logits_to_keep=None, channels=None, ds3_gather_for_generation=True, resume_only_model=False, optimizer=None, loss_type=None, metric=None, eval_use_evalscope=False, eval_dataset=[], eval_dataset_args=None, eval_limit=None, eval_generation_config=None, extra_eval_args=None, use_flash_ckpt=False, sft_alpha=0, train_type='full', local_repo_path=None, galore_config=None)"
382
+ }
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/added_tokens.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "</tool_call>": 151658,
3
+ "<tool_call>": 151657,
4
+ "<|box_end|>": 151649,
5
+ "<|box_start|>": 151648,
6
+ "<|endoftext|>": 151643,
7
+ "<|file_sep|>": 151664,
8
+ "<|fim_middle|>": 151660,
9
+ "<|fim_pad|>": 151662,
10
+ "<|fim_prefix|>": 151659,
11
+ "<|fim_suffix|>": 151661,
12
+ "<|im_end|>": 151645,
13
+ "<|im_start|>": 151644,
14
+ "<|image_pad|>": 151655,
15
+ "<|object_ref_end|>": 151647,
16
+ "<|object_ref_start|>": 151646,
17
+ "<|quad_end|>": 151651,
18
+ "<|quad_start|>": 151650,
19
+ "<|repo_name|>": 151663,
20
+ "<|video_pad|>": 151656,
21
+ "<|vision_end|>": 151653,
22
+ "<|vision_pad|>": 151654,
23
+ "<|vision_start|>": 151652
24
+ }
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/args.json ADDED
@@ -0,0 +1,382 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "output_dir": "/group/40143/hongzhuyi/ms-swift/output/v1-20250912-013615",
3
+ "overwrite_output_dir": false,
4
+ "do_train": false,
5
+ "do_eval": false,
6
+ "do_predict": false,
7
+ "eval_strategy": "epoch",
8
+ "prediction_loss_only": false,
9
+ "per_device_train_batch_size": 2,
10
+ "per_device_eval_batch_size": 1,
11
+ "per_gpu_train_batch_size": null,
12
+ "per_gpu_eval_batch_size": null,
13
+ "gradient_accumulation_steps": 4,
14
+ "eval_accumulation_steps": null,
15
+ "eval_delay": 0,
16
+ "torch_empty_cache_steps": null,
17
+ "learning_rate": 1e-06,
18
+ "weight_decay": 0.1,
19
+ "adam_beta1": 0.9,
20
+ "adam_beta2": 0.95,
21
+ "adam_epsilon": 1e-08,
22
+ "max_grad_norm": 1.0,
23
+ "num_train_epochs": 2.0,
24
+ "max_steps": -1,
25
+ "lr_scheduler_type": "cosine",
26
+ "lr_scheduler_kwargs": null,
27
+ "warmup_ratio": 0.05,
28
+ "warmup_steps": 0,
29
+ "log_level": "passive",
30
+ "log_level_replica": "warning",
31
+ "log_on_each_node": true,
32
+ "logging_dir": "/group/40143/hongzhuyi/ms-swift/output/v1-20250912-013615/runs",
33
+ "logging_strategy": "steps",
34
+ "logging_first_step": true,
35
+ "logging_steps": 1,
36
+ "logging_nan_inf_filter": true,
37
+ "save_strategy": "epoch",
38
+ "save_steps": 500,
39
+ "save_total_limit": null,
40
+ "save_safetensors": true,
41
+ "save_on_each_node": false,
42
+ "save_only_model": false,
43
+ "restore_callback_states_from_checkpoint": false,
44
+ "no_cuda": false,
45
+ "use_cpu": false,
46
+ "use_mps_device": false,
47
+ "seed": 42,
48
+ "data_seed": 42,
49
+ "jit_mode_eval": false,
50
+ "use_ipex": false,
51
+ "bf16": true,
52
+ "fp16": false,
53
+ "fp16_opt_level": "O1",
54
+ "half_precision_backend": "auto",
55
+ "bf16_full_eval": false,
56
+ "fp16_full_eval": false,
57
+ "tf32": null,
58
+ "local_rank": 0,
59
+ "ddp_backend": null,
60
+ "tpu_num_cores": null,
61
+ "tpu_metrics_debug": false,
62
+ "debug": null,
63
+ "dataloader_drop_last": false,
64
+ "eval_steps": 2000.0,
65
+ "dataloader_num_workers": 48,
66
+ "dataloader_prefetch_factor": null,
67
+ "past_index": -1,
68
+ "run_name": "/group/40143/hongzhuyi/ms-swift/output/v1-20250912-013615",
69
+ "disable_tqdm": null,
70
+ "remove_unused_columns": true,
71
+ "label_names": null,
72
+ "load_best_model_at_end": false,
73
+ "metric_for_best_model": "loss",
74
+ "greater_is_better": false,
75
+ "ignore_data_skip": false,
76
+ "fsdp": "",
77
+ "fsdp_min_num_params": 0,
78
+ "fsdp_config": null,
79
+ "fsdp_transformer_layer_cls_to_wrap": null,
80
+ "accelerator_config": {
81
+ "dispatch_batches": false
82
+ },
83
+ "deepspeed": {
84
+ "fp16": {
85
+ "enabled": "auto",
86
+ "loss_scale": 0,
87
+ "loss_scale_window": 1000,
88
+ "initial_scale_power": 16,
89
+ "hysteresis": 2,
90
+ "min_loss_scale": 1
91
+ },
92
+ "bf16": {
93
+ "enabled": "auto"
94
+ },
95
+ "zero_optimization": {
96
+ "stage": 3,
97
+ "offload_optimizer": {
98
+ "device": "none",
99
+ "pin_memory": true
100
+ },
101
+ "offload_param": {
102
+ "device": "none",
103
+ "pin_memory": true
104
+ },
105
+ "overlap_comm": false,
106
+ "contiguous_gradients": true,
107
+ "sub_group_size": 1000000000.0,
108
+ "reduce_bucket_size": "auto",
109
+ "zero_quantized_weights": false,
110
+ "zero_quantized_gradients": false,
111
+ "stage3_prefetch_bucket_size": "auto",
112
+ "stage3_param_persistence_threshold": "auto",
113
+ "stage3_max_live_parameters": 1000000000.0,
114
+ "stage3_max_reuse_distance": 1000000000.0,
115
+ "stage3_gather_16bit_weights_on_model_save": true
116
+ },
117
+ "gradient_accumulation_steps": "auto",
118
+ "gradient_clipping": "auto",
119
+ "steps_per_print": 2000,
120
+ "train_batch_size": "auto",
121
+ "train_micro_batch_size_per_gpu": "auto",
122
+ "wall_clock_breakdown": false
123
+ },
124
+ "label_smoothing_factor": 0.0,
125
+ "optim": "adamw_torch_fused",
126
+ "optim_args": null,
127
+ "adafactor": false,
128
+ "group_by_length": false,
129
+ "length_column_name": "length",
130
+ "report_to": [
131
+ "tensorboard"
132
+ ],
133
+ "ddp_find_unused_parameters": null,
134
+ "ddp_bucket_cap_mb": null,
135
+ "ddp_broadcast_buffers": null,
136
+ "dataloader_pin_memory": true,
137
+ "dataloader_persistent_workers": false,
138
+ "skip_memory_metrics": true,
139
+ "use_legacy_prediction_loop": false,
140
+ "push_to_hub": false,
141
+ "resume_from_checkpoint": null,
142
+ "hub_model_id": null,
143
+ "hub_strategy": "every_save",
144
+ "hub_token": null,
145
+ "hub_private_repo": null,
146
+ "hub_always_push": false,
147
+ "hub_revision": null,
148
+ "gradient_checkpointing": true,
149
+ "gradient_checkpointing_kwargs": null,
150
+ "include_inputs_for_metrics": false,
151
+ "include_for_metrics": [],
152
+ "eval_do_concat_batches": true,
153
+ "fp16_backend": "auto",
154
+ "push_to_hub_model_id": null,
155
+ "push_to_hub_organization": null,
156
+ "push_to_hub_token": null,
157
+ "mp_parameters": "",
158
+ "auto_find_batch_size": false,
159
+ "full_determinism": false,
160
+ "torchdynamo": null,
161
+ "ray_scope": "last",
162
+ "ddp_timeout": 18000000,
163
+ "torch_compile": false,
164
+ "torch_compile_backend": null,
165
+ "torch_compile_mode": null,
166
+ "include_tokens_per_second": false,
167
+ "include_num_input_tokens_seen": false,
168
+ "neftune_noise_alpha": null,
169
+ "optim_target_modules": null,
170
+ "batch_eval_metrics": false,
171
+ "eval_on_start": false,
172
+ "use_liger_kernel": false,
173
+ "liger_kernel_config": null,
174
+ "eval_use_gather_object": false,
175
+ "average_tokens_across_devices": true,
176
+ "sortish_sampler": false,
177
+ "predict_with_generate": false,
178
+ "generation_max_length": null,
179
+ "generation_num_beams": null,
180
+ "generation_config": null,
181
+ "tuner_backend": "peft",
182
+ "vit_gradient_checkpointing": null,
183
+ "router_aux_loss_coef": 0.0,
184
+ "enable_dft_loss": false,
185
+ "check_model": true,
186
+ "acc_strategy": "token",
187
+ "train_dataloader_shuffle": true,
188
+ "max_epochs": null,
189
+ "aligner_lr": null,
190
+ "vit_lr": null,
191
+ "use_logits_to_keep": null,
192
+ "channels": null,
193
+ "ds3_gather_for_generation": true,
194
+ "resume_only_model": false,
195
+ "optimizer": null,
196
+ "loss_type": null,
197
+ "metric": null,
198
+ "eval_use_evalscope": false,
199
+ "eval_dataset": [],
200
+ "eval_dataset_args": null,
201
+ "eval_limit": null,
202
+ "eval_generation_config": null,
203
+ "extra_eval_args": null,
204
+ "use_flash_ckpt": false,
205
+ "model": "Qwen/Qwen2.5-7B-Instruct",
206
+ "model_type": "qwen2_5",
207
+ "model_revision": null,
208
+ "task_type": "causal_lm",
209
+ "torch_dtype": "bfloat16",
210
+ "attn_impl": null,
211
+ "new_special_tokens": [],
212
+ "num_labels": null,
213
+ "problem_type": null,
214
+ "rope_scaling": null,
215
+ "device_map": null,
216
+ "max_memory": {},
217
+ "max_model_len": null,
218
+ "local_repo_path": null,
219
+ "init_strategy": null,
220
+ "template": "qwen2_5",
221
+ "system": null,
222
+ "max_length": 16240,
223
+ "truncation_strategy": "delete",
224
+ "max_pixels": null,
225
+ "agent_template": null,
226
+ "norm_bbox": null,
227
+ "use_chat_template": true,
228
+ "padding_free": false,
229
+ "padding_side": "right",
230
+ "loss_scale": "default",
231
+ "sequence_parallel_size": 1,
232
+ "response_prefix": null,
233
+ "template_backend": "swift",
234
+ "dataset": [
235
+ "./corr_hotpot_2083q_context_swift.jsonl",
236
+ "./corr_hotpot_new1369q_context_swift.jsonl",
237
+ "./corr_nq_2225q_context_swift.jsonl"
238
+ ],
239
+ "val_dataset": [],
240
+ "split_dataset_ratio": 0.001,
241
+ "dataset_num_proc": 100,
242
+ "load_from_cache_file": true,
243
+ "dataset_shuffle": true,
244
+ "val_dataset_shuffle": false,
245
+ "streaming": false,
246
+ "interleave_prob": null,
247
+ "stopping_strategy": "first_exhausted",
248
+ "shuffle_buffer_size": 1000,
249
+ "download_mode": "reuse_dataset_if_exists",
250
+ "columns": {},
251
+ "strict": false,
252
+ "model_name": null,
253
+ "model_author": null,
254
+ "custom_dataset_info": [],
255
+ "quant_method": null,
256
+ "quant_bits": null,
257
+ "hqq_axis": null,
258
+ "bnb_4bit_compute_dtype": "bfloat16",
259
+ "bnb_4bit_quant_type": "nf4",
260
+ "bnb_4bit_use_double_quant": true,
261
+ "bnb_4bit_quant_storage": null,
262
+ "max_new_tokens": 64,
263
+ "temperature": 0.0,
264
+ "top_k": null,
265
+ "top_p": null,
266
+ "repetition_penalty": null,
267
+ "num_beams": 1,
268
+ "stream": false,
269
+ "stop_words": [],
270
+ "logprobs": false,
271
+ "top_logprobs": null,
272
+ "ckpt_dir": null,
273
+ "lora_modules": [],
274
+ "train_type": "full",
275
+ "adapters": [],
276
+ "external_plugins": [],
277
+ "model_kwargs": {},
278
+ "load_args": false,
279
+ "load_data_args": false,
280
+ "packing": false,
281
+ "packing_length": null,
282
+ "lazy_tokenize": false,
283
+ "cached_dataset": [],
284
+ "custom_register_path": [],
285
+ "use_hf": false,
286
+ "ignore_args_error": false,
287
+ "use_swift_lora": false,
288
+ "freeze_parameters": [],
289
+ "freeze_parameters_regex": null,
290
+ "freeze_parameters_ratio": 0.0,
291
+ "trainable_parameters": [],
292
+ "trainable_parameters_regex": null,
293
+ "freeze_llm": false,
294
+ "freeze_vit": true,
295
+ "freeze_aligner": false,
296
+ "target_modules": [
297
+ "all-linear"
298
+ ],
299
+ "target_regex": null,
300
+ "modules_to_save": [],
301
+ "lora_rank": 8,
302
+ "lora_alpha": 32,
303
+ "lora_dropout": 0.05,
304
+ "lora_bias": "none",
305
+ "lora_dtype": null,
306
+ "lorap_lr_ratio": null,
307
+ "use_rslora": false,
308
+ "use_dora": false,
309
+ "lora_ga_batch_size": 2,
310
+ "lora_ga_iters": 2,
311
+ "lora_ga_max_length": 1024,
312
+ "lora_ga_direction": "ArB2r",
313
+ "lora_ga_scale": "stable",
314
+ "lora_ga_stable_gamma": 16,
315
+ "init_weights": true,
316
+ "fourier_n_frequency": 2000,
317
+ "fourier_scaling": 300.0,
318
+ "boft_block_size": 4,
319
+ "boft_block_num": 0,
320
+ "boft_n_butterfly_factor": 1,
321
+ "boft_dropout": 0.0,
322
+ "vera_rank": 256,
323
+ "vera_projection_prng_key": 0,
324
+ "vera_dropout": 0.0,
325
+ "vera_d_initial": 0.1,
326
+ "adapter_act": "gelu",
327
+ "adapter_length": 128,
328
+ "use_galore": false,
329
+ "galore_target_modules": null,
330
+ "galore_rank": 128,
331
+ "galore_update_proj_gap": 50,
332
+ "galore_scale": 1.0,
333
+ "galore_proj_type": "std",
334
+ "galore_optim_per_parameter": false,
335
+ "galore_with_embedding": false,
336
+ "galore_quantization": false,
337
+ "galore_proj_quant": false,
338
+ "galore_proj_bits": 4,
339
+ "galore_proj_group_size": 256,
340
+ "galore_cos_threshold": 0.4,
341
+ "galore_gamma_proj": 2,
342
+ "galore_queue_size": 5,
343
+ "adalora_target_r": 8,
344
+ "adalora_init_r": 12,
345
+ "adalora_tinit": 0,
346
+ "adalora_tfinal": 0,
347
+ "adalora_deltaT": 1,
348
+ "adalora_beta1": 0.85,
349
+ "adalora_beta2": 0.85,
350
+ "adalora_orth_reg_weight": 0.5,
351
+ "llamapro_num_new_blocks": 4,
352
+ "llamapro_num_groups": null,
353
+ "lisa_activated_layers": 0,
354
+ "lisa_step_interval": 20,
355
+ "reft_layer_key": null,
356
+ "reft_layers": null,
357
+ "reft_rank": 4,
358
+ "reft_intervention_type": "LoreftIntervention",
359
+ "reft_args": null,
360
+ "swanlab_token": null,
361
+ "swanlab_project": null,
362
+ "swanlab_workspace": null,
363
+ "swanlab_exp_name": null,
364
+ "swanlab_lark_webhook_url": null,
365
+ "swanlab_lark_secret": null,
366
+ "swanlab_mode": "cloud",
367
+ "add_version": true,
368
+ "create_checkpoint_symlink": false,
369
+ "zero_hpz_partition_size": null,
370
+ "deepspeed_autotp_size": null,
371
+ "early_stop_interval": null,
372
+ "rank": 0,
373
+ "global_world_size": 8,
374
+ "local_world_size": 8,
375
+ "model_suffix": "Qwen2.5-7B-Instruct",
376
+ "model_info": "ModelInfo(model_type='qwen2_5', model_dir='/usr/local/app/.cache/modelscope/hub/models/Qwen/Qwen2___5-7B-Instruct', torch_dtype=torch.bfloat16, max_model_len=32768, quant_method=None, quant_bits=None, rope_scaling=None, is_moe_model=False, config=None, task_type='causal_lm', num_labels=None)",
377
+ "model_meta": "ModelMeta(model_type='qwen2_5', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct', hf_model_id='Qwen/Qwen2.5-3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct', hf_model_id='Qwen/Qwen2.5-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct', hf_model_id='Qwen/Qwen2.5-14B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct', hf_model_id='Qwen/Qwen2.5-32B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct', hf_model_id='Qwen/Qwen2.5-72B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B', hf_model_id='Qwen/Qwen2.5-0.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B', hf_model_id='Qwen/Qwen2.5-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B', hf_model_id='Qwen/Qwen2.5-3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B', hf_model_id='Qwen/Qwen2.5-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B', hf_model_id='Qwen/Qwen2.5-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B', hf_model_id='Qwen/Qwen2.5-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B', hf_model_id='Qwen/Qwen2.5-72B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-3B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-7B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-14B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-32B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-72B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B', hf_model_id='Qwen/Qwen2.5-Coder-0.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B', hf_model_id='Qwen/Qwen2.5-Coder-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B', hf_model_id='Qwen/Qwen2.5-Coder-3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B', hf_model_id='Qwen/Qwen2.5-Coder-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B', hf_model_id='Qwen/Qwen2.5-Coder-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B', hf_model_id='Qwen/Qwen2.5-Coder-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=['coding']), ModelGroup(models=[Model(ms_model_id='moonshotai/Kimi-Dev-72B', hf_model_id='moonshotai/Kimi-Dev-72B', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[])], template='qwen2_5', get_function=<function get_model_tokenizer_with_flash_attn at 0x7f4f5af767a0>, model_arch=ModelKeys(arch_name='llama', embedding='model.embed_tokens', module_list='model.layers', lm_head='lm_head', q_proj='model.layers.{}.self_attn.q_proj', k_proj='model.layers.{}.self_attn.k_proj', v_proj='model.layers.{}.self_attn.v_proj', o_proj='model.layers.{}.self_attn.o_proj', attention='model.layers.{}.self_attn', mlp='model.layers.{}.mlp', down_proj='model.layers.{}.mlp.down_proj', qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None), architectures=['Qwen2ForCausalLM'], additional_saved_files=[], torch_dtype=None, is_multimodal=False, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.37'], tags=[])",
378
+ "model_dir": "/usr/local/app/.cache/modelscope/hub/models/Qwen/Qwen2___5-7B-Instruct",
379
+ "hub": "<class 'swift.hub.hub.MSHub'>",
380
+ "evaluation_strategy": "epoch",
381
+ "training_args": "Seq2SeqTrainingArguments(output_dir='/group/40143/hongzhuyi/ms-swift/output/v1-20250912-013615', overwrite_output_dir=False, do_train=False, do_eval=True, do_predict=False, eval_strategy=<IntervalStrategy.EPOCH: 'epoch'>, prediction_loss_only=False, per_device_train_batch_size=2, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=4, eval_accumulation_steps=None, eval_delay=0, torch_empty_cache_steps=None, learning_rate=1e-06, weight_decay=0.1, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=2.0, max_steps=-1, lr_scheduler_type=<SchedulerType.COSINE: 'cosine'>, lr_scheduler_kwargs=None, warmup_ratio=0.05, warmup_steps=0, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='/group/40143/hongzhuyi/ms-swift/output/v1-20250912-013615/runs', logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_first_step=True, logging_steps=1, logging_nan_inf_filter=True, save_strategy=<SaveStrategy.EPOCH: 'epoch'>, save_steps=500, save_total_limit=None, save_safetensors=True, save_on_each_node=False, save_only_model=False, restore_callback_states_from_checkpoint=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=42, data_seed=42, jit_mode_eval=False, use_ipex=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=2000.0, dataloader_num_workers=48, dataloader_prefetch_factor=10, past_index=-1, run_name='/group/40143/hongzhuyi/ms-swift/output/v1-20250912-013615', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model='loss', greater_is_better=False, ignore_data_skip=False, fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), deepspeed={'fp16': {'enabled': 'auto', 'loss_scale': 0, 'loss_scale_window': 1000, 'initial_scale_power': 16, 'hysteresis': 2, 'min_loss_scale': 1}, 'bf16': {'enabled': 'auto'}, 'zero_optimization': {'stage': 3, 'offload_optimizer': {'device': 'none', 'pin_memory': True}, 'offload_param': {'device': 'none', 'pin_memory': True}, 'overlap_comm': False, 'contiguous_gradients': True, 'sub_group_size': 1000000000.0, 'reduce_bucket_size': 'auto', 'zero_quantized_weights': False, 'zero_quantized_gradients': False, 'stage3_prefetch_bucket_size': 'auto', 'stage3_param_persistence_threshold': 'auto', 'stage3_max_live_parameters': 1000000000.0, 'stage3_max_reuse_distance': 1000000000.0, 'stage3_gather_16bit_weights_on_model_save': True}, 'gradient_accumulation_steps': 'auto', 'gradient_clipping': 'auto', 'steps_per_print': 2000, 'train_batch_size': 'auto', 'train_micro_batch_size_per_gpu': 'auto', 'wall_clock_breakdown': False}, label_smoothing_factor=0.0, optim=<OptimizerNames.ADAMW_TORCH_FUSED: 'adamw_torch_fused'>, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], ddp_find_unused_parameters=None, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, dataloader_persistent_workers=False, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_token=None, hub_private_repo=None, hub_always_push=False, hub_revision=None, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, include_inputs_for_metrics=False, include_for_metrics=[], eval_do_concat_batches=True, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=18000000, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, include_tokens_per_second=None, include_num_input_tokens_seen=None, neftune_noise_alpha=None, optim_target_modules=None, batch_eval_metrics=False, eval_on_start=False, use_liger_kernel=False, liger_kernel_config=None, eval_use_gather_object=False, average_tokens_across_devices=None, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=None, tuner_backend='peft', vit_gradient_checkpointing=True, router_aux_loss_coef=0.0, enable_dft_loss=False, check_model=True, acc_strategy='token', train_dataloader_shuffle=True, max_epochs=None, aligner_lr=None, vit_lr=None, use_logits_to_keep=None, channels=None, ds3_gather_for_generation=True, resume_only_model=False, optimizer=None, loss_type=None, metric=None, eval_use_evalscope=False, eval_dataset=[], eval_dataset_args=None, eval_limit=None, eval_generation_config=None, extra_eval_args=None, use_flash_ckpt=False, sft_alpha=0, train_type='full', local_repo_path=None, galore_config=None)"
382
+ }
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/chat_template.jinja ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {%- if tools %}
2
+ {{- '<|im_start|>system\n' }}
3
+ {%- if messages[0]['role'] == 'system' %}
4
+ {{- messages[0]['content'] }}
5
+ {%- else %}
6
+ {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
7
+ {%- endif %}
8
+ {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
9
+ {%- for tool in tools %}
10
+ {{- "\n" }}
11
+ {{- tool | tojson }}
12
+ {%- endfor %}
13
+ {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
14
+ {%- else %}
15
+ {%- if messages[0]['role'] == 'system' %}
16
+ {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
17
+ {%- else %}
18
+ {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
19
+ {%- endif %}
20
+ {%- endif %}
21
+ {%- for message in messages %}
22
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
23
+ {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
24
+ {%- elif message.role == "assistant" %}
25
+ {{- '<|im_start|>' + message.role }}
26
+ {%- if message.content %}
27
+ {{- '\n' + message.content }}
28
+ {%- endif %}
29
+ {%- for tool_call in message.tool_calls %}
30
+ {%- if tool_call.function is defined %}
31
+ {%- set tool_call = tool_call.function %}
32
+ {%- endif %}
33
+ {{- '\n<tool_call>\n{"name": "' }}
34
+ {{- tool_call.name }}
35
+ {{- '", "arguments": ' }}
36
+ {{- tool_call.arguments | tojson }}
37
+ {{- '}\n</tool_call>' }}
38
+ {%- endfor %}
39
+ {{- '<|im_end|>\n' }}
40
+ {%- elif message.role == "tool" %}
41
+ {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
42
+ {{- '<|im_start|>user' }}
43
+ {%- endif %}
44
+ {{- '\n<tool_response>\n' }}
45
+ {{- message.content }}
46
+ {{- '\n</tool_response>' }}
47
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
48
+ {{- '<|im_end|>\n' }}
49
+ {%- endif %}
50
+ {%- endif %}
51
+ {%- endfor %}
52
+ {%- if add_generation_prompt %}
53
+ {{- '<|im_start|>assistant\n' }}
54
+ {%- endif %}
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/config.json ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen2ForCausalLM"
4
+ ],
5
+ "attention_dropout": 0.0,
6
+ "bos_token_id": 151643,
7
+ "eos_token_id": 151645,
8
+ "hidden_act": "silu",
9
+ "hidden_size": 3584,
10
+ "initializer_range": 0.02,
11
+ "intermediate_size": 18944,
12
+ "layer_types": [
13
+ "full_attention",
14
+ "full_attention",
15
+ "full_attention",
16
+ "full_attention",
17
+ "full_attention",
18
+ "full_attention",
19
+ "full_attention",
20
+ "full_attention",
21
+ "full_attention",
22
+ "full_attention",
23
+ "full_attention",
24
+ "full_attention",
25
+ "full_attention",
26
+ "full_attention",
27
+ "full_attention",
28
+ "full_attention",
29
+ "full_attention",
30
+ "full_attention",
31
+ "full_attention",
32
+ "full_attention",
33
+ "full_attention",
34
+ "full_attention",
35
+ "full_attention",
36
+ "full_attention",
37
+ "full_attention",
38
+ "full_attention",
39
+ "full_attention",
40
+ "full_attention"
41
+ ],
42
+ "max_position_embeddings": 32768,
43
+ "max_window_layers": 28,
44
+ "model_type": "qwen2",
45
+ "num_attention_heads": 28,
46
+ "num_hidden_layers": 28,
47
+ "num_key_value_heads": 4,
48
+ "pad_token_id": 151643,
49
+ "rms_norm_eps": 1e-06,
50
+ "rope_scaling": null,
51
+ "rope_theta": 1000000.0,
52
+ "sliding_window": null,
53
+ "tie_word_embeddings": false,
54
+ "torch_dtype": "bfloat16",
55
+ "transformers_version": "4.55.4",
56
+ "use_cache": false,
57
+ "use_sliding_window": false,
58
+ "vocab_size": 152064
59
+ }
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/generation_config.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 151643,
3
+ "do_sample": true,
4
+ "eos_token_id": [
5
+ 151645,
6
+ 151643
7
+ ],
8
+ "pad_token_id": 151643,
9
+ "repetition_penalty": 1.05,
10
+ "temperature": 0.7,
11
+ "top_k": 20,
12
+ "top_p": 0.8,
13
+ "transformers_version": "4.55.4"
14
+ }
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/latest ADDED
@@ -0,0 +1 @@
 
 
1
+ global_step172
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/model-00001-of-00004.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4165258518f6558699407e0ab66a02cc842df2150d873c91d345cec3897a7f75
3
+ size 4877660776
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/model-00002-of-00004.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7c3f269a93c59c600acbfdb84f8f3010f34837dafa93d5e0fe22f65e318a49ca
3
+ size 4932751008
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/model-00003-of-00004.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6c7ddadf94dc5f3a8dc61b355ed349222450a7cc12b0ed50b6879c69cff1f3e4
3
+ size 4330865200
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/model-00004-of-00004.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:688ea4fcaf405ffe95fbe776546a0d61bed9507aa369e91e5acac52a31a937be
3
+ size 1089994880
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/model.safetensors.index.json ADDED
@@ -0,0 +1,347 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "metadata": {
3
+ "total_parameters": 333312,
4
+ "total_size": 15231233024
5
+ },
6
+ "weight_map": {
7
+ "lm_head.weight": "model-00004-of-00004.safetensors",
8
+ "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
9
+ "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
10
+ "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
11
+ "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
12
+ "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
13
+ "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
14
+ "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
15
+ "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
16
+ "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
17
+ "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
18
+ "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
19
+ "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
20
+ "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
21
+ "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
22
+ "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
23
+ "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
24
+ "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
25
+ "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
26
+ "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
27
+ "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
28
+ "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
29
+ "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
30
+ "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
31
+ "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
32
+ "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
33
+ "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
34
+ "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
35
+ "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
36
+ "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
37
+ "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
38
+ "model.layers.10.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
39
+ "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
40
+ "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
41
+ "model.layers.10.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
42
+ "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
43
+ "model.layers.10.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
44
+ "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
45
+ "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
46
+ "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
47
+ "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
48
+ "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
49
+ "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
50
+ "model.layers.11.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
51
+ "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
52
+ "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
53
+ "model.layers.11.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
54
+ "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
55
+ "model.layers.11.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
56
+ "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
57
+ "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
58
+ "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
59
+ "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
60
+ "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
61
+ "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
62
+ "model.layers.12.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
63
+ "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
64
+ "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
65
+ "model.layers.12.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
66
+ "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
67
+ "model.layers.12.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
68
+ "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
69
+ "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
70
+ "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
71
+ "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
72
+ "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
73
+ "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
74
+ "model.layers.13.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
75
+ "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
76
+ "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
77
+ "model.layers.13.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
78
+ "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
79
+ "model.layers.13.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
80
+ "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
81
+ "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
82
+ "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
83
+ "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
84
+ "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
85
+ "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
86
+ "model.layers.14.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
87
+ "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
88
+ "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
89
+ "model.layers.14.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
90
+ "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
91
+ "model.layers.14.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
92
+ "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
93
+ "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
94
+ "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
95
+ "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
96
+ "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
97
+ "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
98
+ "model.layers.15.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
99
+ "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
100
+ "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
101
+ "model.layers.15.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
102
+ "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
103
+ "model.layers.15.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
104
+ "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
105
+ "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
106
+ "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
107
+ "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
108
+ "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
109
+ "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
110
+ "model.layers.16.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
111
+ "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
112
+ "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
113
+ "model.layers.16.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
114
+ "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
115
+ "model.layers.16.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
116
+ "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
117
+ "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
118
+ "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
119
+ "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
120
+ "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
121
+ "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
122
+ "model.layers.17.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
123
+ "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
124
+ "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
125
+ "model.layers.17.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
126
+ "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
127
+ "model.layers.17.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
128
+ "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
129
+ "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
130
+ "model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
131
+ "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
132
+ "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
133
+ "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
134
+ "model.layers.18.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
135
+ "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
136
+ "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
137
+ "model.layers.18.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
138
+ "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
139
+ "model.layers.18.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
140
+ "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
141
+ "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
142
+ "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
143
+ "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
144
+ "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
145
+ "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
146
+ "model.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
147
+ "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
148
+ "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
149
+ "model.layers.19.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
150
+ "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
151
+ "model.layers.19.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
152
+ "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
153
+ "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
154
+ "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
155
+ "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
156
+ "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
157
+ "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
158
+ "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
159
+ "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
160
+ "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
161
+ "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
162
+ "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
163
+ "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
164
+ "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
165
+ "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
166
+ "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
167
+ "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
168
+ "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
169
+ "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
170
+ "model.layers.20.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
171
+ "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
172
+ "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
173
+ "model.layers.20.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
174
+ "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
175
+ "model.layers.20.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
176
+ "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
177
+ "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
178
+ "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
179
+ "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
180
+ "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
181
+ "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
182
+ "model.layers.21.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
183
+ "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
184
+ "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
185
+ "model.layers.21.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
186
+ "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
187
+ "model.layers.21.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
188
+ "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
189
+ "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
190
+ "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
191
+ "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
192
+ "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
193
+ "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
194
+ "model.layers.22.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
195
+ "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
196
+ "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
197
+ "model.layers.22.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
198
+ "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
199
+ "model.layers.22.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
200
+ "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
201
+ "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
202
+ "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
203
+ "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
204
+ "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
205
+ "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
206
+ "model.layers.23.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
207
+ "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
208
+ "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
209
+ "model.layers.23.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
210
+ "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
211
+ "model.layers.23.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
212
+ "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
213
+ "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
214
+ "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
215
+ "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
216
+ "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
217
+ "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
218
+ "model.layers.24.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
219
+ "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
220
+ "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
221
+ "model.layers.24.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
222
+ "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
223
+ "model.layers.24.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
224
+ "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
225
+ "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
226
+ "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
227
+ "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
228
+ "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
229
+ "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
230
+ "model.layers.25.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
231
+ "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
232
+ "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
233
+ "model.layers.25.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
234
+ "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
235
+ "model.layers.25.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
236
+ "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
237
+ "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
238
+ "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
239
+ "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
240
+ "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
241
+ "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
242
+ "model.layers.26.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
243
+ "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
244
+ "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
245
+ "model.layers.26.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
246
+ "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
247
+ "model.layers.26.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
248
+ "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
249
+ "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
250
+ "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
251
+ "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
252
+ "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
253
+ "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
254
+ "model.layers.27.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
255
+ "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
256
+ "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
257
+ "model.layers.27.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
258
+ "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
259
+ "model.layers.27.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
260
+ "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
261
+ "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
262
+ "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
263
+ "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
264
+ "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
265
+ "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
266
+ "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
267
+ "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
268
+ "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
269
+ "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
270
+ "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
271
+ "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
272
+ "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
273
+ "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
274
+ "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
275
+ "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
276
+ "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
277
+ "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
278
+ "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
279
+ "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
280
+ "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
281
+ "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
282
+ "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
283
+ "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
284
+ "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
285
+ "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
286
+ "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
287
+ "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
288
+ "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
289
+ "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
290
+ "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
291
+ "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
292
+ "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
293
+ "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
294
+ "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
295
+ "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
296
+ "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
297
+ "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
298
+ "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
299
+ "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
300
+ "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
301
+ "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
302
+ "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
303
+ "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
304
+ "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
305
+ "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
306
+ "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
307
+ "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
308
+ "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
309
+ "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
310
+ "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
311
+ "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
312
+ "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
313
+ "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
314
+ "model.layers.7.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
315
+ "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
316
+ "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
317
+ "model.layers.7.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
318
+ "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
319
+ "model.layers.7.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
320
+ "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
321
+ "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
322
+ "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
323
+ "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
324
+ "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
325
+ "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
326
+ "model.layers.8.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
327
+ "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
328
+ "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
329
+ "model.layers.8.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
330
+ "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
331
+ "model.layers.8.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
332
+ "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
333
+ "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
334
+ "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
335
+ "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
336
+ "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
337
+ "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
338
+ "model.layers.9.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
339
+ "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
340
+ "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
341
+ "model.layers.9.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
342
+ "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
343
+ "model.layers.9.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
344
+ "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
345
+ "model.norm.weight": "model-00003-of-00004.safetensors"
346
+ }
347
+ }
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/special_tokens_map.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<|im_start|>",
4
+ "<|im_end|>",
5
+ "<|object_ref_start|>",
6
+ "<|object_ref_end|>",
7
+ "<|box_start|>",
8
+ "<|box_end|>",
9
+ "<|quad_start|>",
10
+ "<|quad_end|>",
11
+ "<|vision_start|>",
12
+ "<|vision_end|>",
13
+ "<|vision_pad|>",
14
+ "<|image_pad|>",
15
+ "<|video_pad|>"
16
+ ],
17
+ "eos_token": {
18
+ "content": "<|im_end|>",
19
+ "lstrip": false,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ },
24
+ "pad_token": {
25
+ "content": "<|endoftext|>",
26
+ "lstrip": false,
27
+ "normalized": false,
28
+ "rstrip": false,
29
+ "single_word": false
30
+ }
31
+ }
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
3
+ size 11421896
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/tokenizer_config.json ADDED
@@ -0,0 +1,207 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": false,
3
+ "add_prefix_space": false,
4
+ "added_tokens_decoder": {
5
+ "151643": {
6
+ "content": "<|endoftext|>",
7
+ "lstrip": false,
8
+ "normalized": false,
9
+ "rstrip": false,
10
+ "single_word": false,
11
+ "special": true
12
+ },
13
+ "151644": {
14
+ "content": "<|im_start|>",
15
+ "lstrip": false,
16
+ "normalized": false,
17
+ "rstrip": false,
18
+ "single_word": false,
19
+ "special": true
20
+ },
21
+ "151645": {
22
+ "content": "<|im_end|>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false,
27
+ "special": true
28
+ },
29
+ "151646": {
30
+ "content": "<|object_ref_start|>",
31
+ "lstrip": false,
32
+ "normalized": false,
33
+ "rstrip": false,
34
+ "single_word": false,
35
+ "special": true
36
+ },
37
+ "151647": {
38
+ "content": "<|object_ref_end|>",
39
+ "lstrip": false,
40
+ "normalized": false,
41
+ "rstrip": false,
42
+ "single_word": false,
43
+ "special": true
44
+ },
45
+ "151648": {
46
+ "content": "<|box_start|>",
47
+ "lstrip": false,
48
+ "normalized": false,
49
+ "rstrip": false,
50
+ "single_word": false,
51
+ "special": true
52
+ },
53
+ "151649": {
54
+ "content": "<|box_end|>",
55
+ "lstrip": false,
56
+ "normalized": false,
57
+ "rstrip": false,
58
+ "single_word": false,
59
+ "special": true
60
+ },
61
+ "151650": {
62
+ "content": "<|quad_start|>",
63
+ "lstrip": false,
64
+ "normalized": false,
65
+ "rstrip": false,
66
+ "single_word": false,
67
+ "special": true
68
+ },
69
+ "151651": {
70
+ "content": "<|quad_end|>",
71
+ "lstrip": false,
72
+ "normalized": false,
73
+ "rstrip": false,
74
+ "single_word": false,
75
+ "special": true
76
+ },
77
+ "151652": {
78
+ "content": "<|vision_start|>",
79
+ "lstrip": false,
80
+ "normalized": false,
81
+ "rstrip": false,
82
+ "single_word": false,
83
+ "special": true
84
+ },
85
+ "151653": {
86
+ "content": "<|vision_end|>",
87
+ "lstrip": false,
88
+ "normalized": false,
89
+ "rstrip": false,
90
+ "single_word": false,
91
+ "special": true
92
+ },
93
+ "151654": {
94
+ "content": "<|vision_pad|>",
95
+ "lstrip": false,
96
+ "normalized": false,
97
+ "rstrip": false,
98
+ "single_word": false,
99
+ "special": true
100
+ },
101
+ "151655": {
102
+ "content": "<|image_pad|>",
103
+ "lstrip": false,
104
+ "normalized": false,
105
+ "rstrip": false,
106
+ "single_word": false,
107
+ "special": true
108
+ },
109
+ "151656": {
110
+ "content": "<|video_pad|>",
111
+ "lstrip": false,
112
+ "normalized": false,
113
+ "rstrip": false,
114
+ "single_word": false,
115
+ "special": true
116
+ },
117
+ "151657": {
118
+ "content": "<tool_call>",
119
+ "lstrip": false,
120
+ "normalized": false,
121
+ "rstrip": false,
122
+ "single_word": false,
123
+ "special": false
124
+ },
125
+ "151658": {
126
+ "content": "</tool_call>",
127
+ "lstrip": false,
128
+ "normalized": false,
129
+ "rstrip": false,
130
+ "single_word": false,
131
+ "special": false
132
+ },
133
+ "151659": {
134
+ "content": "<|fim_prefix|>",
135
+ "lstrip": false,
136
+ "normalized": false,
137
+ "rstrip": false,
138
+ "single_word": false,
139
+ "special": false
140
+ },
141
+ "151660": {
142
+ "content": "<|fim_middle|>",
143
+ "lstrip": false,
144
+ "normalized": false,
145
+ "rstrip": false,
146
+ "single_word": false,
147
+ "special": false
148
+ },
149
+ "151661": {
150
+ "content": "<|fim_suffix|>",
151
+ "lstrip": false,
152
+ "normalized": false,
153
+ "rstrip": false,
154
+ "single_word": false,
155
+ "special": false
156
+ },
157
+ "151662": {
158
+ "content": "<|fim_pad|>",
159
+ "lstrip": false,
160
+ "normalized": false,
161
+ "rstrip": false,
162
+ "single_word": false,
163
+ "special": false
164
+ },
165
+ "151663": {
166
+ "content": "<|repo_name|>",
167
+ "lstrip": false,
168
+ "normalized": false,
169
+ "rstrip": false,
170
+ "single_word": false,
171
+ "special": false
172
+ },
173
+ "151664": {
174
+ "content": "<|file_sep|>",
175
+ "lstrip": false,
176
+ "normalized": false,
177
+ "rstrip": false,
178
+ "single_word": false,
179
+ "special": false
180
+ }
181
+ },
182
+ "additional_special_tokens": [
183
+ "<|im_start|>",
184
+ "<|im_end|>",
185
+ "<|object_ref_start|>",
186
+ "<|object_ref_end|>",
187
+ "<|box_start|>",
188
+ "<|box_end|>",
189
+ "<|quad_start|>",
190
+ "<|quad_end|>",
191
+ "<|vision_start|>",
192
+ "<|vision_end|>",
193
+ "<|vision_pad|>",
194
+ "<|image_pad|>",
195
+ "<|video_pad|>"
196
+ ],
197
+ "bos_token": null,
198
+ "clean_up_tokenization_spaces": false,
199
+ "eos_token": "<|im_end|>",
200
+ "errors": "replace",
201
+ "extra_special_tokens": {},
202
+ "model_max_length": 131072,
203
+ "pad_token": "<|endoftext|>",
204
+ "split_special_tokens": false,
205
+ "tokenizer_class": "Qwen2Tokenizer",
206
+ "unk_token": null
207
+ }
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/trainer_state.json ADDED
@@ -0,0 +1,1437 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": 172,
3
+ "best_metric": 0.36040598,
4
+ "best_model_checkpoint": "/group/40143/hongzhuyi/ms-swift/output/v1-20250912-013615/checkpoint-172",
5
+ "epoch": 2.0,
6
+ "eval_steps": 2000.0,
7
+ "global_step": 172,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.011730205278592375,
14
+ "grad_norm": 19.42347886416143,
15
+ "learning_rate": 1.111111111111111e-07,
16
+ "loss": 0.9745279550552368,
17
+ "step": 1,
18
+ "token_acc": 0.7729915976524353
19
+ },
20
+ {
21
+ "epoch": 0.02346041055718475,
22
+ "grad_norm": 18.20102937211775,
23
+ "learning_rate": 2.222222222222222e-07,
24
+ "loss": 0.9701757431030273,
25
+ "step": 2,
26
+ "token_acc": 0.7739322185516357
27
+ },
28
+ {
29
+ "epoch": 0.03519061583577713,
30
+ "grad_norm": 19.78533819918654,
31
+ "learning_rate": 3.333333333333333e-07,
32
+ "loss": 1.0109223127365112,
33
+ "step": 3,
34
+ "token_acc": 0.7666413187980652
35
+ },
36
+ {
37
+ "epoch": 0.0469208211143695,
38
+ "grad_norm": 19.03486317488165,
39
+ "learning_rate": 4.444444444444444e-07,
40
+ "loss": 0.9887339472770691,
41
+ "step": 4,
42
+ "token_acc": 0.7682322859764099
43
+ },
44
+ {
45
+ "epoch": 0.05865102639296188,
46
+ "grad_norm": 17.64436521081935,
47
+ "learning_rate": 5.555555555555555e-07,
48
+ "loss": 0.94492107629776,
49
+ "step": 5,
50
+ "token_acc": 0.775278627872467
51
+ },
52
+ {
53
+ "epoch": 0.07038123167155426,
54
+ "grad_norm": 17.48642955713519,
55
+ "learning_rate": 6.666666666666666e-07,
56
+ "loss": 0.9331197738647461,
57
+ "step": 6,
58
+ "token_acc": 0.7808876037597656
59
+ },
60
+ {
61
+ "epoch": 0.08211143695014662,
62
+ "grad_norm": 17.401940797372863,
63
+ "learning_rate": 7.777777777777778e-07,
64
+ "loss": 0.9543712139129639,
65
+ "step": 7,
66
+ "token_acc": 0.7736433148384094
67
+ },
68
+ {
69
+ "epoch": 0.093841642228739,
70
+ "grad_norm": 14.675874021386344,
71
+ "learning_rate": 8.888888888888888e-07,
72
+ "loss": 0.9072513580322266,
73
+ "step": 8,
74
+ "token_acc": 0.7740758061408997
75
+ },
76
+ {
77
+ "epoch": 0.10557184750733138,
78
+ "grad_norm": 15.006000189143862,
79
+ "learning_rate": 1e-06,
80
+ "loss": 0.9155452251434326,
81
+ "step": 9,
82
+ "token_acc": 0.7733547687530518
83
+ },
84
+ {
85
+ "epoch": 0.11730205278592376,
86
+ "grad_norm": 9.152052949663721,
87
+ "learning_rate": 9.999071352056673e-07,
88
+ "loss": 0.8092969059944153,
89
+ "step": 10,
90
+ "token_acc": 0.7808704972267151
91
+ },
92
+ {
93
+ "epoch": 0.12903225806451613,
94
+ "grad_norm": 8.326679281131945,
95
+ "learning_rate": 9.996285753181497e-07,
96
+ "loss": 0.7987140417098999,
97
+ "step": 11,
98
+ "token_acc": 0.7814286947250366
99
+ },
100
+ {
101
+ "epoch": 0.14076246334310852,
102
+ "grad_norm": 7.439884752413428,
103
+ "learning_rate": 9.99164423811074e-07,
104
+ "loss": 0.7931578159332275,
105
+ "step": 12,
106
+ "token_acc": 0.7772960066795349
107
+ },
108
+ {
109
+ "epoch": 0.15249266862170088,
110
+ "grad_norm": 7.119023890418445,
111
+ "learning_rate": 9.985148530977764e-07,
112
+ "loss": 0.7600352764129639,
113
+ "step": 13,
114
+ "token_acc": 0.7844690680503845
115
+ },
116
+ {
117
+ "epoch": 0.16422287390029325,
118
+ "grad_norm": 5.035964456988531,
119
+ "learning_rate": 9.976801044672607e-07,
120
+ "loss": 0.6961469650268555,
121
+ "step": 14,
122
+ "token_acc": 0.7958953976631165
123
+ },
124
+ {
125
+ "epoch": 0.17595307917888564,
126
+ "grad_norm": 5.1530837703132075,
127
+ "learning_rate": 9.966604879945656e-07,
128
+ "loss": 0.6827518343925476,
129
+ "step": 15,
130
+ "token_acc": 0.7969183921813965
131
+ },
132
+ {
133
+ "epoch": 0.187683284457478,
134
+ "grad_norm": 4.578053887212758,
135
+ "learning_rate": 9.954563824255877e-07,
136
+ "loss": 0.6470961570739746,
137
+ "step": 16,
138
+ "token_acc": 0.8064683079719543
139
+ },
140
+ {
141
+ "epoch": 0.19941348973607037,
142
+ "grad_norm": 4.471874336838139,
143
+ "learning_rate": 9.94068235036391e-07,
144
+ "loss": 0.6799627542495728,
145
+ "step": 17,
146
+ "token_acc": 0.8002287149429321
147
+ },
148
+ {
149
+ "epoch": 0.21114369501466276,
150
+ "grad_norm": 4.5065795595929705,
151
+ "learning_rate": 9.924965614670628e-07,
152
+ "loss": 0.6805820465087891,
153
+ "step": 18,
154
+ "token_acc": 0.8004816770553589
155
+ },
156
+ {
157
+ "epoch": 0.22287390029325513,
158
+ "grad_norm": 4.026774039902112,
159
+ "learning_rate": 9.90741945530174e-07,
160
+ "loss": 0.6293234825134277,
161
+ "step": 19,
162
+ "token_acc": 0.8133951425552368
163
+ },
164
+ {
165
+ "epoch": 0.23460410557184752,
166
+ "grad_norm": 4.1944187169445035,
167
+ "learning_rate": 9.888050389939172e-07,
168
+ "loss": 0.6192097067832947,
169
+ "step": 20,
170
+ "token_acc": 0.8118802905082703
171
+ },
172
+ {
173
+ "epoch": 0.24633431085043989,
174
+ "grad_norm": 3.826379198058098,
175
+ "learning_rate": 9.866865613400006e-07,
176
+ "loss": 0.6224480867385864,
177
+ "step": 21,
178
+ "token_acc": 0.8149859309196472
179
+ },
180
+ {
181
+ "epoch": 0.25806451612903225,
182
+ "grad_norm": 3.7571032828466873,
183
+ "learning_rate": 9.843872994963912e-07,
184
+ "loss": 0.655880331993103,
185
+ "step": 22,
186
+ "token_acc": 0.8066907525062561
187
+ },
188
+ {
189
+ "epoch": 0.2697947214076246,
190
+ "grad_norm": 3.681409260856744,
191
+ "learning_rate": 9.819081075450013e-07,
192
+ "loss": 0.5886544585227966,
193
+ "step": 23,
194
+ "token_acc": 0.8210309743881226
195
+ },
196
+ {
197
+ "epoch": 0.28152492668621704,
198
+ "grad_norm": 3.8664356748850275,
199
+ "learning_rate": 9.792499064044342e-07,
200
+ "loss": 0.5898054838180542,
201
+ "step": 24,
202
+ "token_acc": 0.819602906703949
203
+ },
204
+ {
205
+ "epoch": 0.2932551319648094,
206
+ "grad_norm": 3.769342936970688,
207
+ "learning_rate": 9.764136834878985e-07,
208
+ "loss": 0.5778634548187256,
209
+ "step": 25,
210
+ "token_acc": 0.8214534521102905
211
+ },
212
+ {
213
+ "epoch": 0.30498533724340177,
214
+ "grad_norm": 3.311750534405761,
215
+ "learning_rate": 9.734004923364256e-07,
216
+ "loss": 0.5714818239212036,
217
+ "step": 26,
218
+ "token_acc": 0.8214825987815857
219
+ },
220
+ {
221
+ "epoch": 0.31671554252199413,
222
+ "grad_norm": 3.2183065918077927,
223
+ "learning_rate": 9.702114522275216e-07,
224
+ "loss": 0.5601825714111328,
225
+ "step": 27,
226
+ "token_acc": 0.8292351365089417
227
+ },
228
+ {
229
+ "epoch": 0.3284457478005865,
230
+ "grad_norm": 2.7897603795961583,
231
+ "learning_rate": 9.66847747759402e-07,
232
+ "loss": 0.527702808380127,
233
+ "step": 28,
234
+ "token_acc": 0.835481584072113
235
+ },
236
+ {
237
+ "epoch": 0.34017595307917886,
238
+ "grad_norm": 2.847650852465056,
239
+ "learning_rate": 9.63310628410961e-07,
240
+ "loss": 0.5497344136238098,
241
+ "step": 29,
242
+ "token_acc": 0.8285976648330688
243
+ },
244
+ {
245
+ "epoch": 0.3519061583577713,
246
+ "grad_norm": 2.705846258459626,
247
+ "learning_rate": 9.596014080776421e-07,
248
+ "loss": 0.50850510597229,
249
+ "step": 30,
250
+ "token_acc": 0.8413625359535217
251
+ },
252
+ {
253
+ "epoch": 0.36363636363636365,
254
+ "grad_norm": 2.4259522658265134,
255
+ "learning_rate": 9.55721464583379e-07,
256
+ "loss": 0.5382215976715088,
257
+ "step": 31,
258
+ "token_acc": 0.8332553505897522
259
+ },
260
+ {
261
+ "epoch": 0.375366568914956,
262
+ "grad_norm": 2.571450563187992,
263
+ "learning_rate": 9.516722391687902e-07,
264
+ "loss": 0.5567100644111633,
265
+ "step": 32,
266
+ "token_acc": 0.825095534324646
267
+ },
268
+ {
269
+ "epoch": 0.3870967741935484,
270
+ "grad_norm": 2.4023357046065903,
271
+ "learning_rate": 9.474552359558165e-07,
272
+ "loss": 0.5305403470993042,
273
+ "step": 33,
274
+ "token_acc": 0.8326757550239563
275
+ },
276
+ {
277
+ "epoch": 0.39882697947214074,
278
+ "grad_norm": 2.3633434913754563,
279
+ "learning_rate": 9.430720213890029e-07,
280
+ "loss": 0.5253998041152954,
281
+ "step": 34,
282
+ "token_acc": 0.8343205451965332
283
+ },
284
+ {
285
+ "epoch": 0.41055718475073316,
286
+ "grad_norm": 2.4155773004795615,
287
+ "learning_rate": 9.385242236536259e-07,
288
+ "loss": 0.5054779052734375,
289
+ "step": 35,
290
+ "token_acc": 0.8360969424247742
291
+ },
292
+ {
293
+ "epoch": 0.4222873900293255,
294
+ "grad_norm": 2.1668550281375953,
295
+ "learning_rate": 9.338135320708911e-07,
296
+ "loss": 0.5100793838500977,
297
+ "step": 36,
298
+ "token_acc": 0.8403894305229187
299
+ },
300
+ {
301
+ "epoch": 0.4340175953079179,
302
+ "grad_norm": 2.2142817682644336,
303
+ "learning_rate": 9.289416964704185e-07,
304
+ "loss": 0.5320375561714172,
305
+ "step": 37,
306
+ "token_acc": 0.8320334553718567
307
+ },
308
+ {
309
+ "epoch": 0.44574780058651026,
310
+ "grad_norm": 2.231366943821694,
311
+ "learning_rate": 9.239105265402525e-07,
312
+ "loss": 0.5052040815353394,
313
+ "step": 38,
314
+ "token_acc": 0.8394160866737366
315
+ },
316
+ {
317
+ "epoch": 0.4574780058651026,
318
+ "grad_norm": 2.1947282253307785,
319
+ "learning_rate": 9.187218911546361e-07,
320
+ "loss": 0.5045727491378784,
321
+ "step": 39,
322
+ "token_acc": 0.8403294086456299
323
+ },
324
+ {
325
+ "epoch": 0.46920821114369504,
326
+ "grad_norm": 2.082686891153019,
327
+ "learning_rate": 9.133777176798012e-07,
328
+ "loss": 0.4821414649486542,
329
+ "step": 40,
330
+ "token_acc": 0.8482713103294373
331
+ },
332
+ {
333
+ "epoch": 0.4809384164222874,
334
+ "grad_norm": 2.1147451850371293,
335
+ "learning_rate": 9.078799912580303e-07,
336
+ "loss": 0.49139469861984253,
337
+ "step": 41,
338
+ "token_acc": 0.8435467481613159
339
+ },
340
+ {
341
+ "epoch": 0.49266862170087977,
342
+ "grad_norm": 2.0542615188891578,
343
+ "learning_rate": 9.022307540702576e-07,
344
+ "loss": 0.495099276304245,
345
+ "step": 42,
346
+ "token_acc": 0.8379960656166077
347
+ },
348
+ {
349
+ "epoch": 0.5043988269794721,
350
+ "grad_norm": 2.0136030676549392,
351
+ "learning_rate": 8.964321045774806e-07,
352
+ "loss": 0.489553302526474,
353
+ "step": 43,
354
+ "token_acc": 0.8405314087867737
355
+ },
356
+ {
357
+ "epoch": 0.5161290322580645,
358
+ "grad_norm": 1.9630127736815466,
359
+ "learning_rate": 8.904861967412701e-07,
360
+ "loss": 0.4687340557575226,
361
+ "step": 44,
362
+ "token_acc": 0.8476594090461731
363
+ },
364
+ {
365
+ "epoch": 0.5278592375366569,
366
+ "grad_norm": 1.9513540796416378,
367
+ "learning_rate": 8.843952392236593e-07,
368
+ "loss": 0.4688946306705475,
369
+ "step": 45,
370
+ "token_acc": 0.8495379090309143
371
+ },
372
+ {
373
+ "epoch": 0.5395894428152492,
374
+ "grad_norm": 2.0000887958335443,
375
+ "learning_rate": 8.781614945667168e-07,
376
+ "loss": 0.459961861371994,
377
+ "step": 46,
378
+ "token_acc": 0.8521719574928284
379
+ },
380
+ {
381
+ "epoch": 0.5513196480938416,
382
+ "grad_norm": 1.9642568842781296,
383
+ "learning_rate": 8.717872783521047e-07,
384
+ "loss": 0.48166409134864807,
385
+ "step": 47,
386
+ "token_acc": 0.8472545146942139
387
+ },
388
+ {
389
+ "epoch": 0.5630498533724341,
390
+ "grad_norm": 1.9787796147862529,
391
+ "learning_rate": 8.652749583409339e-07,
392
+ "loss": 0.47355878353118896,
393
+ "step": 48,
394
+ "token_acc": 0.8466734886169434
395
+ },
396
+ {
397
+ "epoch": 0.5747800586510264,
398
+ "grad_norm": 1.9308160364727027,
399
+ "learning_rate": 8.586269535942385e-07,
400
+ "loss": 0.4618028998374939,
401
+ "step": 49,
402
+ "token_acc": 0.8513069152832031
403
+ },
404
+ {
405
+ "epoch": 0.5865102639296188,
406
+ "grad_norm": 1.8562690832047652,
407
+ "learning_rate": 8.518457335743924e-07,
408
+ "loss": 0.4850979447364807,
409
+ "step": 50,
410
+ "token_acc": 0.8428641557693481
411
+ },
412
+ {
413
+ "epoch": 0.5982404692082112,
414
+ "grad_norm": 1.8213744008266877,
415
+ "learning_rate": 8.449338172278058e-07,
416
+ "loss": 0.4308124780654907,
417
+ "step": 51,
418
+ "token_acc": 0.8571488857269287
419
+ },
420
+ {
421
+ "epoch": 0.6099706744868035,
422
+ "grad_norm": 1.9045019401040395,
423
+ "learning_rate": 8.378937720492383e-07,
424
+ "loss": 0.4853397607803345,
425
+ "step": 52,
426
+ "token_acc": 0.8429522514343262
427
+ },
428
+ {
429
+ "epoch": 0.6217008797653959,
430
+ "grad_norm": 1.8218123277369342,
431
+ "learning_rate": 8.307282131280804e-07,
432
+ "loss": 0.4712865352630615,
433
+ "step": 53,
434
+ "token_acc": 0.8480793833732605
435
+ },
436
+ {
437
+ "epoch": 0.6334310850439883,
438
+ "grad_norm": 1.8207444397924821,
439
+ "learning_rate": 8.23439802176954e-07,
440
+ "loss": 0.4410493075847626,
441
+ "step": 54,
442
+ "token_acc": 0.85650634765625
443
+ },
444
+ {
445
+ "epoch": 0.6451612903225806,
446
+ "grad_norm": 1.7646262400395012,
447
+ "learning_rate": 8.160312465429952e-07,
448
+ "loss": 0.45616358518600464,
449
+ "step": 55,
450
+ "token_acc": 0.8514736890792847
451
+ },
452
+ {
453
+ "epoch": 0.656891495601173,
454
+ "grad_norm": 1.859319205117003,
455
+ "learning_rate": 8.085052982021847e-07,
456
+ "loss": 0.4499448537826538,
457
+ "step": 56,
458
+ "token_acc": 0.8548539876937866
459
+ },
460
+ {
461
+ "epoch": 0.6686217008797654,
462
+ "grad_norm": 1.6613176216022334,
463
+ "learning_rate": 8.008647527371022e-07,
464
+ "loss": 0.43093234300613403,
465
+ "step": 57,
466
+ "token_acc": 0.8587403893470764
467
+ },
468
+ {
469
+ "epoch": 0.6803519061583577,
470
+ "grad_norm": 1.7104406984196343,
471
+ "learning_rate": 7.931124482984801e-07,
472
+ "loss": 0.46069544553756714,
473
+ "step": 58,
474
+ "token_acc": 0.8481629490852356
475
+ },
476
+ {
477
+ "epoch": 0.6920821114369502,
478
+ "grad_norm": 1.8009260213577374,
479
+ "learning_rate": 7.85251264550948e-07,
480
+ "loss": 0.45625898241996765,
481
+ "step": 59,
482
+ "token_acc": 0.8500686287879944
483
+ },
484
+ {
485
+ "epoch": 0.7038123167155426,
486
+ "grad_norm": 1.699888333456424,
487
+ "learning_rate": 7.772841216033532e-07,
488
+ "loss": 0.46078723669052124,
489
+ "step": 60,
490
+ "token_acc": 0.8482969403266907
491
+ },
492
+ {
493
+ "epoch": 0.7155425219941349,
494
+ "grad_norm": 1.738809601837965,
495
+ "learning_rate": 7.69213978924061e-07,
496
+ "loss": 0.4371066987514496,
497
+ "step": 61,
498
+ "token_acc": 0.8570924401283264
499
+ },
500
+ {
501
+ "epoch": 0.7272727272727273,
502
+ "grad_norm": 1.6763935168681205,
503
+ "learning_rate": 7.610438342416319e-07,
504
+ "loss": 0.4535030722618103,
505
+ "step": 62,
506
+ "token_acc": 0.8520426750183105
507
+ },
508
+ {
509
+ "epoch": 0.7390029325513197,
510
+ "grad_norm": 1.6971447873397694,
511
+ "learning_rate": 7.527767224312882e-07,
512
+ "loss": 0.4411541521549225,
513
+ "step": 63,
514
+ "token_acc": 0.854651153087616
515
+ },
516
+ {
517
+ "epoch": 0.750733137829912,
518
+ "grad_norm": 1.7785564815624373,
519
+ "learning_rate": 7.444157143875819e-07,
520
+ "loss": 0.43432003259658813,
521
+ "step": 64,
522
+ "token_acc": 0.8572342395782471
523
+ },
524
+ {
525
+ "epoch": 0.7624633431085044,
526
+ "grad_norm": 1.7516774946326659,
527
+ "learning_rate": 7.359639158836827e-07,
528
+ "loss": 0.4576050937175751,
529
+ "step": 65,
530
+ "token_acc": 0.852763831615448
531
+ },
532
+ {
533
+ "epoch": 0.7741935483870968,
534
+ "grad_norm": 1.8159173350443534,
535
+ "learning_rate": 7.274244664177097e-07,
536
+ "loss": 0.4335535764694214,
537
+ "step": 66,
538
+ "token_acc": 0.8581987023353577
539
+ },
540
+ {
541
+ "epoch": 0.7859237536656891,
542
+ "grad_norm": 1.6985715890277628,
543
+ "learning_rate": 7.188005380465364e-07,
544
+ "loss": 0.43045544624328613,
545
+ "step": 67,
546
+ "token_acc": 0.8622540831565857
547
+ },
548
+ {
549
+ "epoch": 0.7976539589442815,
550
+ "grad_norm": 1.6425422999936439,
551
+ "learning_rate": 7.100953342075009e-07,
552
+ "loss": 0.4037867486476898,
553
+ "step": 68,
554
+ "token_acc": 0.8682027459144592
555
+ },
556
+ {
557
+ "epoch": 0.8093841642228738,
558
+ "grad_norm": 1.729981054732428,
559
+ "learning_rate": 7.013120885284598e-07,
560
+ "loss": 0.45598796010017395,
561
+ "step": 69,
562
+ "token_acc": 0.8521715998649597
563
+ },
564
+ {
565
+ "epoch": 0.8211143695014663,
566
+ "grad_norm": 1.7637112650059394,
567
+ "learning_rate": 6.924540636266272e-07,
568
+ "loss": 0.44459569454193115,
569
+ "step": 70,
570
+ "token_acc": 0.854467511177063
571
+ },
572
+ {
573
+ "epoch": 0.8328445747800587,
574
+ "grad_norm": 1.7887656982582423,
575
+ "learning_rate": 6.83524549896646e-07,
576
+ "loss": 0.45404496788978577,
577
+ "step": 71,
578
+ "token_acc": 0.851087749004364
579
+ },
580
+ {
581
+ "epoch": 0.844574780058651,
582
+ "grad_norm": 1.6852945606800631,
583
+ "learning_rate": 6.745268642883404e-07,
584
+ "loss": 0.4606991708278656,
585
+ "step": 72,
586
+ "token_acc": 0.8486180305480957
587
+ },
588
+ {
589
+ "epoch": 0.8563049853372434,
590
+ "grad_norm": 1.6399899629621875,
591
+ "learning_rate": 6.654643490746041e-07,
592
+ "loss": 0.45651188492774963,
593
+ "step": 73,
594
+ "token_acc": 0.8511216640472412
595
+ },
596
+ {
597
+ "epoch": 0.8680351906158358,
598
+ "grad_norm": 1.738549808935775,
599
+ "learning_rate": 6.563403706098832e-07,
600
+ "loss": 0.44438982009887695,
601
+ "step": 74,
602
+ "token_acc": 0.8562652468681335
603
+ },
604
+ {
605
+ "epoch": 0.8797653958944281,
606
+ "grad_norm": 1.7269017599894254,
607
+ "learning_rate": 6.47158318079712e-07,
608
+ "loss": 0.41442689299583435,
609
+ "step": 75,
610
+ "token_acc": 0.8645210862159729
611
+ },
612
+ {
613
+ "epoch": 0.8914956011730205,
614
+ "grad_norm": 1.7743815830820935,
615
+ "learning_rate": 6.379216022417695e-07,
616
+ "loss": 0.42937588691711426,
617
+ "step": 76,
618
+ "token_acc": 0.8610733151435852
619
+ },
620
+ {
621
+ "epoch": 0.9032258064516129,
622
+ "grad_norm": 1.5857889981196076,
623
+ "learning_rate": 6.286336541589223e-07,
624
+ "loss": 0.4185515344142914,
625
+ "step": 77,
626
+ "token_acc": 0.863837718963623
627
+ },
628
+ {
629
+ "epoch": 0.9149560117302052,
630
+ "grad_norm": 1.6775726033449394,
631
+ "learning_rate": 6.192979239247242e-07,
632
+ "loss": 0.39629387855529785,
633
+ "step": 78,
634
+ "token_acc": 0.8690212368965149
635
+ },
636
+ {
637
+ "epoch": 0.9266862170087976,
638
+ "grad_norm": 1.6671136884891102,
639
+ "learning_rate": 6.099178793818478e-07,
640
+ "loss": 0.4317322075366974,
641
+ "step": 79,
642
+ "token_acc": 0.8624503016471863
643
+ },
644
+ {
645
+ "epoch": 0.9384164222873901,
646
+ "grad_norm": 1.6320376815696551,
647
+ "learning_rate": 6.004970048339225e-07,
648
+ "loss": 0.4439336657524109,
649
+ "step": 80,
650
+ "token_acc": 0.8561707139015198
651
+ },
652
+ {
653
+ "epoch": 0.9501466275659824,
654
+ "grad_norm": 1.6010567395710715,
655
+ "learning_rate": 5.910387997512573e-07,
656
+ "loss": 0.43129977583885193,
657
+ "step": 81,
658
+ "token_acc": 0.8600221872329712
659
+ },
660
+ {
661
+ "epoch": 0.9618768328445748,
662
+ "grad_norm": 1.6137708142260028,
663
+ "learning_rate": 5.815467774709313e-07,
664
+ "loss": 0.40753841400146484,
665
+ "step": 82,
666
+ "token_acc": 0.8669683337211609
667
+ },
668
+ {
669
+ "epoch": 0.9736070381231672,
670
+ "grad_norm": 1.71404162588074,
671
+ "learning_rate": 5.720244638917323e-07,
672
+ "loss": 0.40983232855796814,
673
+ "step": 83,
674
+ "token_acc": 0.8663516044616699
675
+ },
676
+ {
677
+ "epoch": 0.9853372434017595,
678
+ "grad_norm": 1.7287418122113773,
679
+ "learning_rate": 5.624753961644281e-07,
680
+ "loss": 0.43514540791511536,
681
+ "step": 84,
682
+ "token_acc": 0.8586166501045227
683
+ },
684
+ {
685
+ "epoch": 0.9970674486803519,
686
+ "grad_norm": 1.7083837850323478,
687
+ "learning_rate": 5.529031213778614e-07,
688
+ "loss": 0.4181726574897766,
689
+ "step": 85,
690
+ "token_acc": 0.8610889911651611
691
+ },
692
+ {
693
+ "epoch": 1.0,
694
+ "grad_norm": 4.682383086910253,
695
+ "learning_rate": 5.433111952413494e-07,
696
+ "loss": 0.437732994556427,
697
+ "step": 86,
698
+ "token_acc": 0.8590138554573059
699
+ },
700
+ {
701
+ "epoch": 1.0,
702
+ "eval_loss": 0.3737797141075134,
703
+ "eval_runtime": 4.3026,
704
+ "eval_samples_per_second": 1.162,
705
+ "eval_steps_per_second": 0.232,
706
+ "eval_token_acc": 0.8611544370651245,
707
+ "step": 86
708
+ },
709
+ {
710
+ "epoch": 1.0117302052785924,
711
+ "grad_norm": 1.5677714188723684,
712
+ "learning_rate": 5.33703180763884e-07,
713
+ "loss": 0.3829854130744934,
714
+ "step": 87,
715
+ "token_acc": 0.8707565069198608
716
+ },
717
+ {
718
+ "epoch": 1.0234604105571847,
719
+ "grad_norm": 1.5225303880357206,
720
+ "learning_rate": 5.240826469306186e-07,
721
+ "loss": 0.41871583461761475,
722
+ "step": 88,
723
+ "token_acc": 0.861519992351532
724
+ },
725
+ {
726
+ "epoch": 1.035190615835777,
727
+ "grad_norm": 1.6345919572102399,
728
+ "learning_rate": 5.144531673771363e-07,
729
+ "loss": 0.4087834358215332,
730
+ "step": 89,
731
+ "token_acc": 0.8636031150817871
732
+ },
733
+ {
734
+ "epoch": 1.0469208211143695,
735
+ "grad_norm": 1.655708414041218,
736
+ "learning_rate": 5.048183190619903e-07,
737
+ "loss": 0.41285789012908936,
738
+ "step": 90,
739
+ "token_acc": 0.863472044467926
740
+ },
741
+ {
742
+ "epoch": 1.0586510263929618,
743
+ "grad_norm": 1.6138271383053973,
744
+ "learning_rate": 4.951816809380097e-07,
745
+ "loss": 0.4109683632850647,
746
+ "step": 91,
747
+ "token_acc": 0.8626707196235657
748
+ },
749
+ {
750
+ "epoch": 1.0703812316715542,
751
+ "grad_norm": 1.565760777471466,
752
+ "learning_rate": 4.855468326228638e-07,
753
+ "loss": 0.4262546896934509,
754
+ "step": 92,
755
+ "token_acc": 0.8588240146636963
756
+ },
757
+ {
758
+ "epoch": 1.0821114369501466,
759
+ "grad_norm": 1.6679466399894174,
760
+ "learning_rate": 4.7591735306938134e-07,
761
+ "loss": 0.41767850518226624,
762
+ "step": 93,
763
+ "token_acc": 0.8628318309783936
764
+ },
765
+ {
766
+ "epoch": 1.093841642228739,
767
+ "grad_norm": 1.5255876664856023,
768
+ "learning_rate": 4.6629681923611603e-07,
769
+ "loss": 0.39251255989074707,
770
+ "step": 94,
771
+ "token_acc": 0.8693065047264099
772
+ },
773
+ {
774
+ "epoch": 1.1055718475073313,
775
+ "grad_norm": 1.6676418700538884,
776
+ "learning_rate": 4.5668880475865067e-07,
777
+ "loss": 0.42066100239753723,
778
+ "step": 95,
779
+ "token_acc": 0.8595956563949585
780
+ },
781
+ {
782
+ "epoch": 1.1173020527859236,
783
+ "grad_norm": 1.6720418292335593,
784
+ "learning_rate": 4.4709687862213864e-07,
785
+ "loss": 0.4325845241546631,
786
+ "step": 96,
787
+ "token_acc": 0.858229398727417
788
+ },
789
+ {
790
+ "epoch": 1.129032258064516,
791
+ "grad_norm": 1.5696471790513795,
792
+ "learning_rate": 4.3752460383557194e-07,
793
+ "loss": 0.37914514541625977,
794
+ "step": 97,
795
+ "token_acc": 0.8733330368995667
796
+ },
797
+ {
798
+ "epoch": 1.1407624633431086,
799
+ "grad_norm": 1.6722377519465854,
800
+ "learning_rate": 4.2797553610826797e-07,
801
+ "loss": 0.421178936958313,
802
+ "step": 98,
803
+ "token_acc": 0.859768807888031
804
+ },
805
+ {
806
+ "epoch": 1.152492668621701,
807
+ "grad_norm": 1.5811341006573918,
808
+ "learning_rate": 4.184532225290686e-07,
809
+ "loss": 0.40062060952186584,
810
+ "step": 99,
811
+ "token_acc": 0.8644114136695862
812
+ },
813
+ {
814
+ "epoch": 1.1642228739002933,
815
+ "grad_norm": 1.7146569707668204,
816
+ "learning_rate": 4.089612002487428e-07,
817
+ "loss": 0.42175137996673584,
818
+ "step": 100,
819
+ "token_acc": 0.8623378872871399
820
+ },
821
+ {
822
+ "epoch": 1.1759530791788857,
823
+ "grad_norm": 1.6226235337966886,
824
+ "learning_rate": 3.995029951660776e-07,
825
+ "loss": 0.4274284839630127,
826
+ "step": 101,
827
+ "token_acc": 0.8579118847846985
828
+ },
829
+ {
830
+ "epoch": 1.187683284457478,
831
+ "grad_norm": 1.54879470169086,
832
+ "learning_rate": 3.9008212061815207e-07,
833
+ "loss": 0.42116108536720276,
834
+ "step": 102,
835
+ "token_acc": 0.8611886501312256
836
+ },
837
+ {
838
+ "epoch": 1.1994134897360704,
839
+ "grad_norm": 1.5492835608458386,
840
+ "learning_rate": 3.8070207607527585e-07,
841
+ "loss": 0.4289548695087433,
842
+ "step": 103,
843
+ "token_acc": 0.8600953221321106
844
+ },
845
+ {
846
+ "epoch": 1.2111436950146628,
847
+ "grad_norm": 1.5754100747832267,
848
+ "learning_rate": 3.7136634584107783e-07,
849
+ "loss": 0.4085935950279236,
850
+ "step": 104,
851
+ "token_acc": 0.8644001483917236
852
+ },
853
+ {
854
+ "epoch": 1.2228739002932552,
855
+ "grad_norm": 1.6029423160607676,
856
+ "learning_rate": 3.6207839775823047e-07,
857
+ "loss": 0.42785000801086426,
858
+ "step": 105,
859
+ "token_acc": 0.8579254746437073
860
+ },
861
+ {
862
+ "epoch": 1.2346041055718475,
863
+ "grad_norm": 1.5804032250182314,
864
+ "learning_rate": 3.5284168192028805e-07,
865
+ "loss": 0.40198612213134766,
866
+ "step": 106,
867
+ "token_acc": 0.8686321973800659
868
+ },
869
+ {
870
+ "epoch": 1.2463343108504399,
871
+ "grad_norm": 1.5752581280233084,
872
+ "learning_rate": 3.4365962939011693e-07,
873
+ "loss": 0.4137642979621887,
874
+ "step": 107,
875
+ "token_acc": 0.8617110848426819
876
+ },
877
+ {
878
+ "epoch": 1.2580645161290323,
879
+ "grad_norm": 1.5480325492274043,
880
+ "learning_rate": 3.345356509253958e-07,
881
+ "loss": 0.4173002243041992,
882
+ "step": 108,
883
+ "token_acc": 0.861508309841156
884
+ },
885
+ {
886
+ "epoch": 1.2697947214076246,
887
+ "grad_norm": 1.5186055094039825,
888
+ "learning_rate": 3.2547313571165967e-07,
889
+ "loss": 0.3934311866760254,
890
+ "step": 109,
891
+ "token_acc": 0.867075502872467
892
+ },
893
+ {
894
+ "epoch": 1.281524926686217,
895
+ "grad_norm": 1.5310007026213002,
896
+ "learning_rate": 3.1647545010335395e-07,
897
+ "loss": 0.4025633633136749,
898
+ "step": 110,
899
+ "token_acc": 0.8660922050476074
900
+ },
901
+ {
902
+ "epoch": 1.2932551319648093,
903
+ "grad_norm": 1.577615399058995,
904
+ "learning_rate": 3.075459363733727e-07,
905
+ "loss": 0.39246582984924316,
906
+ "step": 111,
907
+ "token_acc": 0.8689733147621155
908
+ },
909
+ {
910
+ "epoch": 1.3049853372434017,
911
+ "grad_norm": 1.4272120854730381,
912
+ "learning_rate": 2.9868791147154025e-07,
913
+ "loss": 0.4092977046966553,
914
+ "step": 112,
915
+ "token_acc": 0.8651414513587952
916
+ },
917
+ {
918
+ "epoch": 1.316715542521994,
919
+ "grad_norm": 1.625119604585716,
920
+ "learning_rate": 2.8990466579249917e-07,
921
+ "loss": 0.42311206459999084,
922
+ "step": 113,
923
+ "token_acc": 0.860737144947052
924
+ },
925
+ {
926
+ "epoch": 1.3284457478005864,
927
+ "grad_norm": 1.6863637660172988,
928
+ "learning_rate": 2.811994619534637e-07,
929
+ "loss": 0.4120984375476837,
930
+ "step": 114,
931
+ "token_acc": 0.8642065525054932
932
+ },
933
+ {
934
+ "epoch": 1.3401759530791788,
935
+ "grad_norm": 1.6289238939686508,
936
+ "learning_rate": 2.725755335822903e-07,
937
+ "loss": 0.3722948431968689,
938
+ "step": 115,
939
+ "token_acc": 0.8729490041732788
940
+ },
941
+ {
942
+ "epoch": 1.3519061583577714,
943
+ "grad_norm": 1.6624092640178676,
944
+ "learning_rate": 2.640360841163174e-07,
945
+ "loss": 0.4013710916042328,
946
+ "step": 116,
947
+ "token_acc": 0.8644599318504333
948
+ },
949
+ {
950
+ "epoch": 1.3636363636363638,
951
+ "grad_norm": 1.6129470586326828,
952
+ "learning_rate": 2.5558428561241816e-07,
953
+ "loss": 0.4010535180568695,
954
+ "step": 117,
955
+ "token_acc": 0.8661304712295532
956
+ },
957
+ {
958
+ "epoch": 1.3753665689149561,
959
+ "grad_norm": 1.5515066116434881,
960
+ "learning_rate": 2.472232775687119e-07,
961
+ "loss": 0.38552382588386536,
962
+ "step": 118,
963
+ "token_acc": 0.8697531819343567
964
+ },
965
+ {
966
+ "epoch": 1.3870967741935485,
967
+ "grad_norm": 1.5467230515129131,
968
+ "learning_rate": 2.3895616575836806e-07,
969
+ "loss": 0.4148092269897461,
970
+ "step": 119,
971
+ "token_acc": 0.8629264831542969
972
+ },
973
+ {
974
+ "epoch": 1.3988269794721409,
975
+ "grad_norm": 1.5834947597509632,
976
+ "learning_rate": 2.3078602107593897e-07,
977
+ "loss": 0.3911811113357544,
978
+ "step": 120,
979
+ "token_acc": 0.868228018283844
980
+ },
981
+ {
982
+ "epoch": 1.4105571847507332,
983
+ "grad_norm": 1.6141986405617998,
984
+ "learning_rate": 2.2271587839664668e-07,
985
+ "loss": 0.3904862105846405,
986
+ "step": 121,
987
+ "token_acc": 0.87054842710495
988
+ },
989
+ {
990
+ "epoch": 1.4222873900293256,
991
+ "grad_norm": 1.6191892226375433,
992
+ "learning_rate": 2.1474873544905203e-07,
993
+ "loss": 0.39509063959121704,
994
+ "step": 122,
995
+ "token_acc": 0.8696496486663818
996
+ },
997
+ {
998
+ "epoch": 1.434017595307918,
999
+ "grad_norm": 1.6077908719984266,
1000
+ "learning_rate": 2.0688755170151994e-07,
1001
+ "loss": 0.3951858878135681,
1002
+ "step": 123,
1003
+ "token_acc": 0.8712750673294067
1004
+ },
1005
+ {
1006
+ "epoch": 1.4457478005865103,
1007
+ "grad_norm": 1.5216947856270064,
1008
+ "learning_rate": 1.991352472628978e-07,
1009
+ "loss": 0.3887852430343628,
1010
+ "step": 124,
1011
+ "token_acc": 0.8729190826416016
1012
+ },
1013
+ {
1014
+ "epoch": 1.4574780058651027,
1015
+ "grad_norm": 1.6042869016843018,
1016
+ "learning_rate": 1.9149470179781529e-07,
1017
+ "loss": 0.41031360626220703,
1018
+ "step": 125,
1019
+ "token_acc": 0.8616762161254883
1020
+ },
1021
+ {
1022
+ "epoch": 1.469208211143695,
1023
+ "grad_norm": 1.6337299865347379,
1024
+ "learning_rate": 1.8396875345700496e-07,
1025
+ "loss": 0.4053962528705597,
1026
+ "step": 126,
1027
+ "token_acc": 0.8648815751075745
1028
+ },
1029
+ {
1030
+ "epoch": 1.4809384164222874,
1031
+ "grad_norm": 1.7038782786631552,
1032
+ "learning_rate": 1.76560197823046e-07,
1033
+ "loss": 0.3860664367675781,
1034
+ "step": 127,
1035
+ "token_acc": 0.8715552687644958
1036
+ },
1037
+ {
1038
+ "epoch": 1.4926686217008798,
1039
+ "grad_norm": 1.5209677554470171,
1040
+ "learning_rate": 1.6927178687191952e-07,
1041
+ "loss": 0.41598591208457947,
1042
+ "step": 128,
1043
+ "token_acc": 0.8615263104438782
1044
+ },
1045
+ {
1046
+ "epoch": 1.5043988269794721,
1047
+ "grad_norm": 1.5489353954928298,
1048
+ "learning_rate": 1.6210622795076167e-07,
1049
+ "loss": 0.4063546657562256,
1050
+ "step": 129,
1051
+ "token_acc": 0.8664236664772034
1052
+ },
1053
+ {
1054
+ "epoch": 1.5161290322580645,
1055
+ "grad_norm": 1.4774729636438646,
1056
+ "learning_rate": 1.5506618277219408e-07,
1057
+ "loss": 0.4048060178756714,
1058
+ "step": 130,
1059
+ "token_acc": 0.8650010228157043
1060
+ },
1061
+ {
1062
+ "epoch": 1.5278592375366569,
1063
+ "grad_norm": 1.5553515644243028,
1064
+ "learning_rate": 1.481542664256075e-07,
1065
+ "loss": 0.44169881939888,
1066
+ "step": 131,
1067
+ "token_acc": 0.8539696931838989
1068
+ },
1069
+ {
1070
+ "epoch": 1.5395894428152492,
1071
+ "grad_norm": 1.6320796136309983,
1072
+ "learning_rate": 1.413730464057616e-07,
1073
+ "loss": 0.38520485162734985,
1074
+ "step": 132,
1075
+ "token_acc": 0.8700256943702698
1076
+ },
1077
+ {
1078
+ "epoch": 1.5513196480938416,
1079
+ "grad_norm": 1.7213472526760363,
1080
+ "learning_rate": 1.3472504165906612e-07,
1081
+ "loss": 0.38589727878570557,
1082
+ "step": 133,
1083
+ "token_acc": 0.8720390200614929
1084
+ },
1085
+ {
1086
+ "epoch": 1.563049853372434,
1087
+ "grad_norm": 1.61612025336049,
1088
+ "learning_rate": 1.2821272164789543e-07,
1089
+ "loss": 0.3985021114349365,
1090
+ "step": 134,
1091
+ "token_acc": 0.8674476146697998
1092
+ },
1093
+ {
1094
+ "epoch": 1.5747800586510263,
1095
+ "grad_norm": 1.6129753719216395,
1096
+ "learning_rate": 1.2183850543328312e-07,
1097
+ "loss": 0.4011296033859253,
1098
+ "step": 135,
1099
+ "token_acc": 0.8660385608673096
1100
+ },
1101
+ {
1102
+ "epoch": 1.5865102639296187,
1103
+ "grad_norm": 1.6609518421068834,
1104
+ "learning_rate": 1.1560476077634069e-07,
1105
+ "loss": 0.400723397731781,
1106
+ "step": 136,
1107
+ "token_acc": 0.8661333322525024
1108
+ },
1109
+ {
1110
+ "epoch": 1.598240469208211,
1111
+ "grad_norm": 1.6174608108392161,
1112
+ "learning_rate": 1.0951380325872977e-07,
1113
+ "loss": 0.3947940468788147,
1114
+ "step": 137,
1115
+ "token_acc": 0.8677906394004822
1116
+ },
1117
+ {
1118
+ "epoch": 1.6099706744868034,
1119
+ "grad_norm": 1.5803935684069241,
1120
+ "learning_rate": 1.0356789542251936e-07,
1121
+ "loss": 0.3855525851249695,
1122
+ "step": 138,
1123
+ "token_acc": 0.8687975406646729
1124
+ },
1125
+ {
1126
+ "epoch": 1.6217008797653958,
1127
+ "grad_norm": 1.5602442793936881,
1128
+ "learning_rate": 9.776924592974256e-08,
1129
+ "loss": 0.3888525664806366,
1130
+ "step": 139,
1131
+ "token_acc": 0.8693127036094666
1132
+ },
1133
+ {
1134
+ "epoch": 1.6334310850439882,
1135
+ "grad_norm": 1.6221900032838974,
1136
+ "learning_rate": 9.212000874196952e-08,
1137
+ "loss": 0.40530452132225037,
1138
+ "step": 140,
1139
+ "token_acc": 0.8655845522880554
1140
+ },
1141
+ {
1142
+ "epoch": 1.6451612903225805,
1143
+ "grad_norm": 1.6504808121233552,
1144
+ "learning_rate": 8.662228232019875e-08,
1145
+ "loss": 0.3893311619758606,
1146
+ "step": 141,
1147
+ "token_acc": 0.8734939694404602
1148
+ },
1149
+ {
1150
+ "epoch": 1.6568914956011729,
1151
+ "grad_norm": 1.6200715699558466,
1152
+ "learning_rate": 8.127810884536402e-08,
1153
+ "loss": 0.4028111696243286,
1154
+ "step": 142,
1155
+ "token_acc": 0.8652303218841553
1156
+ },
1157
+ {
1158
+ "epoch": 1.6686217008797652,
1159
+ "grad_norm": 1.5943441640197737,
1160
+ "learning_rate": 7.608947345974759e-08,
1161
+ "loss": 0.4057155251502991,
1162
+ "step": 143,
1163
+ "token_acc": 0.8663085103034973
1164
+ },
1165
+ {
1166
+ "epoch": 1.6803519061583576,
1167
+ "grad_norm": 1.5700703050134148,
1168
+ "learning_rate": 7.105830352958142e-08,
1169
+ "loss": 0.4034878611564636,
1170
+ "step": 144,
1171
+ "token_acc": 0.867348313331604
1172
+ },
1173
+ {
1174
+ "epoch": 1.6920821114369502,
1175
+ "grad_norm": 1.5673287734486288,
1176
+ "learning_rate": 6.618646792910893e-08,
1177
+ "loss": 0.3811131715774536,
1178
+ "step": 145,
1179
+ "token_acc": 0.8729737997055054
1180
+ },
1181
+ {
1182
+ "epoch": 1.7038123167155426,
1183
+ "grad_norm": 1.6603076877822582,
1184
+ "learning_rate": 6.147577634637413e-08,
1185
+ "loss": 0.4155927896499634,
1186
+ "step": 146,
1187
+ "token_acc": 0.8620749711990356
1188
+ },
1189
+ {
1190
+ "epoch": 1.715542521994135,
1191
+ "grad_norm": 1.5668199603698232,
1192
+ "learning_rate": 5.692797861099718e-08,
1193
+ "loss": 0.4296550750732422,
1194
+ "step": 147,
1195
+ "token_acc": 0.8596217036247253
1196
+ },
1197
+ {
1198
+ "epoch": 1.7272727272727273,
1199
+ "grad_norm": 1.4976238860494637,
1200
+ "learning_rate": 5.25447640441834e-08,
1201
+ "loss": 0.40992194414138794,
1202
+ "step": 148,
1203
+ "token_acc": 0.8648180365562439
1204
+ },
1205
+ {
1206
+ "epoch": 1.7390029325513197,
1207
+ "grad_norm": 1.5660992804405987,
1208
+ "learning_rate": 4.832776083120982e-08,
1209
+ "loss": 0.40360546112060547,
1210
+ "step": 149,
1211
+ "token_acc": 0.8669781684875488
1212
+ },
1213
+ {
1214
+ "epoch": 1.750733137829912,
1215
+ "grad_norm": 1.5395912091553665,
1216
+ "learning_rate": 4.427853541662091e-08,
1217
+ "loss": 0.4034501910209656,
1218
+ "step": 150,
1219
+ "token_acc": 0.8668680787086487
1220
+ },
1221
+ {
1222
+ "epoch": 1.7624633431085044,
1223
+ "grad_norm": 1.535539296687079,
1224
+ "learning_rate": 4.039859192235778e-08,
1225
+ "loss": 0.4094376862049103,
1226
+ "step": 151,
1227
+ "token_acc": 0.8650097846984863
1228
+ },
1229
+ {
1230
+ "epoch": 1.7741935483870968,
1231
+ "grad_norm": 1.4812137514743298,
1232
+ "learning_rate": 3.668937158903901e-08,
1233
+ "loss": 0.37502023577690125,
1234
+ "step": 152,
1235
+ "token_acc": 0.8744542002677917
1236
+ },
1237
+ {
1238
+ "epoch": 1.7859237536656891,
1239
+ "grad_norm": 1.4960830542100994,
1240
+ "learning_rate": 3.3152252240598086e-08,
1241
+ "loss": 0.3876444101333618,
1242
+ "step": 153,
1243
+ "token_acc": 0.8692108988761902
1244
+ },
1245
+ {
1246
+ "epoch": 1.7976539589442815,
1247
+ "grad_norm": 1.585776494852321,
1248
+ "learning_rate": 2.978854777247841e-08,
1249
+ "loss": 0.3956642150878906,
1250
+ "step": 154,
1251
+ "token_acc": 0.8652946352958679
1252
+ },
1253
+ {
1254
+ "epoch": 1.8093841642228738,
1255
+ "grad_norm": 1.7274072967614538,
1256
+ "learning_rate": 2.6599507663574384e-08,
1257
+ "loss": 0.39082279801368713,
1258
+ "step": 155,
1259
+ "token_acc": 0.8702662587165833
1260
+ },
1261
+ {
1262
+ "epoch": 1.8211143695014664,
1263
+ "grad_norm": 1.4895866146765766,
1264
+ "learning_rate": 2.358631651210141e-08,
1265
+ "loss": 0.3919299244880676,
1266
+ "step": 156,
1267
+ "token_acc": 0.8669038414955139
1268
+ },
1269
+ {
1270
+ "epoch": 1.8328445747800588,
1271
+ "grad_norm": 1.6136036060416228,
1272
+ "learning_rate": 2.0750093595565733e-08,
1273
+ "loss": 0.38754791021347046,
1274
+ "step": 157,
1275
+ "token_acc": 0.8719207048416138
1276
+ },
1277
+ {
1278
+ "epoch": 1.8445747800586512,
1279
+ "grad_norm": 1.5332743029215397,
1280
+ "learning_rate": 1.8091892454998593e-08,
1281
+ "loss": 0.38901928067207336,
1282
+ "step": 158,
1283
+ "token_acc": 0.8713217377662659
1284
+ },
1285
+ {
1286
+ "epoch": 1.8563049853372435,
1287
+ "grad_norm": 1.7104982698701636,
1288
+ "learning_rate": 1.5612700503608967e-08,
1289
+ "loss": 0.3915766477584839,
1290
+ "step": 159,
1291
+ "token_acc": 0.8659932613372803
1292
+ },
1293
+ {
1294
+ "epoch": 1.868035190615836,
1295
+ "grad_norm": 1.5740451699960079,
1296
+ "learning_rate": 1.3313438659999399e-08,
1297
+ "loss": 0.41466644406318665,
1298
+ "step": 160,
1299
+ "token_acc": 0.8601726293563843
1300
+ },
1301
+ {
1302
+ "epoch": 1.8797653958944283,
1303
+ "grad_norm": 1.4935208181756303,
1304
+ "learning_rate": 1.119496100608297e-08,
1305
+ "loss": 0.4020458459854126,
1306
+ "step": 161,
1307
+ "token_acc": 0.8664253950119019
1308
+ },
1309
+ {
1310
+ "epoch": 1.8914956011730206,
1311
+ "grad_norm": 1.587820321975359,
1312
+ "learning_rate": 9.258054469825972e-09,
1313
+ "loss": 0.39795613288879395,
1314
+ "step": 162,
1315
+ "token_acc": 0.8671923875808716
1316
+ },
1317
+ {
1318
+ "epoch": 1.903225806451613,
1319
+ "grad_norm": 1.655929959372447,
1320
+ "learning_rate": 7.503438532937168e-09,
1321
+ "loss": 0.4237522780895233,
1322
+ "step": 163,
1323
+ "token_acc": 0.8580747246742249
1324
+ },
1325
+ {
1326
+ "epoch": 1.9149560117302054,
1327
+ "grad_norm": 1.5438059638706878,
1328
+ "learning_rate": 5.931764963608865e-09,
1329
+ "loss": 0.38159292936325073,
1330
+ "step": 164,
1331
+ "token_acc": 0.8721365332603455
1332
+ },
1333
+ {
1334
+ "epoch": 1.9266862170087977,
1335
+ "grad_norm": 1.5298862114811087,
1336
+ "learning_rate": 4.543617574412184e-09,
1337
+ "loss": 0.3859734535217285,
1338
+ "step": 165,
1339
+ "token_acc": 0.8722429871559143
1340
+ },
1341
+ {
1342
+ "epoch": 1.93841642228739,
1343
+ "grad_norm": 1.5586225464292018,
1344
+ "learning_rate": 3.3395120054343086e-09,
1345
+ "loss": 0.3822712302207947,
1346
+ "step": 166,
1347
+ "token_acc": 0.8720780611038208
1348
+ },
1349
+ {
1350
+ "epoch": 1.9501466275659824,
1351
+ "grad_norm": 1.4166358748591938,
1352
+ "learning_rate": 2.3198955327393686e-09,
1353
+ "loss": 0.38301122188568115,
1354
+ "step": 167,
1355
+ "token_acc": 0.8709322214126587
1356
+ },
1357
+ {
1358
+ "epoch": 1.9618768328445748,
1359
+ "grad_norm": 1.5351941009075238,
1360
+ "learning_rate": 1.4851469022233997e-09,
1361
+ "loss": 0.3990333080291748,
1362
+ "step": 168,
1363
+ "token_acc": 0.8678142428398132
1364
+ },
1365
+ {
1366
+ "epoch": 1.9736070381231672,
1367
+ "grad_norm": 1.4980089019040588,
1368
+ "learning_rate": 8.35576188926046e-10,
1369
+ "loss": 0.38594484329223633,
1370
+ "step": 169,
1371
+ "token_acc": 0.8707625269889832
1372
+ },
1373
+ {
1374
+ "epoch": 1.9853372434017595,
1375
+ "grad_norm": 1.5452409060373131,
1376
+ "learning_rate": 3.71424681850141e-10,
1377
+ "loss": 0.40693551301956177,
1378
+ "step": 170,
1379
+ "token_acc": 0.8644877076148987
1380
+ },
1381
+ {
1382
+ "epoch": 1.997067448680352,
1383
+ "grad_norm": 1.6283425656352981,
1384
+ "learning_rate": 9.286479433257e-11,
1385
+ "loss": 0.40735697746276855,
1386
+ "step": 171,
1387
+ "token_acc": 0.8651864528656006
1388
+ },
1389
+ {
1390
+ "epoch": 2.0,
1391
+ "grad_norm": 4.506046802107319,
1392
+ "learning_rate": 0.0,
1393
+ "loss": 0.42119351029396057,
1394
+ "step": 172,
1395
+ "token_acc": 0.8619781732559204
1396
+ },
1397
+ {
1398
+ "epoch": 2.0,
1399
+ "eval_loss": 0.36040598154067993,
1400
+ "eval_runtime": 4.558,
1401
+ "eval_samples_per_second": 1.097,
1402
+ "eval_steps_per_second": 0.219,
1403
+ "eval_token_acc": 0.864586591720581,
1404
+ "step": 172
1405
+ },
1406
+ {
1407
+ "epoch": 2.0,
1408
+ "eval_loss": 0.36040598154067993,
1409
+ "eval_runtime": 7.752,
1410
+ "eval_samples_per_second": 0.645,
1411
+ "eval_steps_per_second": 0.129,
1412
+ "eval_token_acc": 0.864586591720581,
1413
+ "step": 172
1414
+ }
1415
+ ],
1416
+ "logging_steps": 1,
1417
+ "max_steps": 172,
1418
+ "num_input_tokens_seen": 0,
1419
+ "num_train_epochs": 2,
1420
+ "save_steps": 500,
1421
+ "stateful_callbacks": {
1422
+ "TrainerControl": {
1423
+ "args": {
1424
+ "should_epoch_stop": false,
1425
+ "should_evaluate": false,
1426
+ "should_log": false,
1427
+ "should_save": true,
1428
+ "should_training_stop": true
1429
+ },
1430
+ "attributes": {}
1431
+ }
1432
+ },
1433
+ "total_flos": 202031623626752.0,
1434
+ "train_batch_size": 2,
1435
+ "trial_name": null,
1436
+ "trial_params": null
1437
+ }
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:59069f1b6b198eaea104863a28f5a6f6ab3f67712636ca0246f3165e68c62374
3
+ size 9041
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/vocab.json ADDED
The diff for this file is too large to render. See raw diff
 
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-172/zero_to_fp32.py ADDED
@@ -0,0 +1,760 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python
2
+
3
+ # Copyright (c) Microsoft Corporation.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ # DeepSpeed Team
7
+
8
+ # This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
9
+ # copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
10
+ # the future. Once extracted, the weights don't require DeepSpeed and can be used in any
11
+ # application.
12
+ #
13
+ # example:
14
+ # python zero_to_fp32.py . output_dir/
15
+ # or
16
+ # python zero_to_fp32.py . output_dir/ --safe_serialization
17
+
18
+ import argparse
19
+ import torch
20
+ import glob
21
+ import math
22
+ import os
23
+ import re
24
+ import gc
25
+ import json
26
+ import numpy as np
27
+ from tqdm import tqdm
28
+ from collections import OrderedDict
29
+ from dataclasses import dataclass
30
+
31
+ # while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
32
+ # DeepSpeed data structures it has to be available in the current python environment.
33
+ from deepspeed.utils import logger
34
+ from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
35
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
36
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
37
+
38
+
39
+ @dataclass
40
+ class zero_model_state:
41
+ buffers: dict()
42
+ param_shapes: dict()
43
+ shared_params: list
44
+ ds_version: int
45
+ frozen_param_shapes: dict()
46
+ frozen_param_fragments: dict()
47
+
48
+
49
+ debug = 0
50
+
51
+ # load to cpu
52
+ device = torch.device('cpu')
53
+
54
+
55
+ def atoi(text):
56
+ return int(text) if text.isdigit() else text
57
+
58
+
59
+ def natural_keys(text):
60
+ '''
61
+ alist.sort(key=natural_keys) sorts in human order
62
+ http://nedbatchelder.com/blog/200712/human_sorting.html
63
+ (See Toothy's implementation in the comments)
64
+ '''
65
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
66
+
67
+
68
+ def get_model_state_file(checkpoint_dir, zero_stage):
69
+ if not os.path.isdir(checkpoint_dir):
70
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
71
+
72
+ # there should be only one file
73
+ if zero_stage <= 2:
74
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
75
+ elif zero_stage == 3:
76
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
77
+
78
+ if not os.path.exists(file):
79
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
80
+
81
+ return file
82
+
83
+
84
+ def get_checkpoint_files(checkpoint_dir, glob_pattern):
85
+ # XXX: need to test that this simple glob rule works for multi-node setup too
86
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
87
+
88
+ if len(ckpt_files) == 0:
89
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
90
+
91
+ return ckpt_files
92
+
93
+
94
+ def get_optim_files(checkpoint_dir):
95
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
96
+
97
+
98
+ def get_model_state_files(checkpoint_dir):
99
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
100
+
101
+
102
+ def parse_model_states(files):
103
+ zero_model_states = []
104
+ for file in files:
105
+ state_dict = torch.load(file, map_location=device, weights_only=False)
106
+
107
+ if BUFFER_NAMES not in state_dict:
108
+ raise ValueError(f"{file} is not a model state checkpoint")
109
+ buffer_names = state_dict[BUFFER_NAMES]
110
+ if debug:
111
+ print("Found buffers:", buffer_names)
112
+
113
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
114
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
115
+ param_shapes = state_dict[PARAM_SHAPES]
116
+
117
+ # collect parameters that are included in param_shapes
118
+ param_names = []
119
+ for s in param_shapes:
120
+ for name in s.keys():
121
+ param_names.append(name)
122
+
123
+ # update with frozen parameters
124
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
125
+ if frozen_param_shapes is not None:
126
+ if debug:
127
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
128
+ param_names += list(frozen_param_shapes.keys())
129
+
130
+ # handle shared params
131
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
132
+
133
+ ds_version = state_dict.get(DS_VERSION, None)
134
+
135
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
136
+
137
+ z_model_state = zero_model_state(buffers=buffers,
138
+ param_shapes=param_shapes,
139
+ shared_params=shared_params,
140
+ ds_version=ds_version,
141
+ frozen_param_shapes=frozen_param_shapes,
142
+ frozen_param_fragments=frozen_param_fragments)
143
+ zero_model_states.append(z_model_state)
144
+
145
+ return zero_model_states
146
+
147
+
148
+ def parse_optim_states(files, ds_checkpoint_dir):
149
+ total_files = len(files)
150
+ state_dicts = []
151
+ for f in tqdm(files, desc='Loading checkpoint shards'):
152
+ state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
153
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
154
+ # and also handle the case where it was already removed by another helper script
155
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
156
+ state_dicts.append(state_dict)
157
+
158
+ if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
159
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
160
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
161
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
162
+
163
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
164
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
165
+ # use the max of the partition_count to get the dp world_size.
166
+
167
+ if type(world_size) is list:
168
+ world_size = max(world_size)
169
+
170
+ if world_size != total_files:
171
+ raise ValueError(
172
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
173
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
174
+ )
175
+
176
+ # the groups are named differently in each stage
177
+ if zero_stage <= 2:
178
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
179
+ elif zero_stage == 3:
180
+ fp32_groups_key = FP32_FLAT_GROUPS
181
+ else:
182
+ raise ValueError(f"unknown zero stage {zero_stage}")
183
+
184
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
185
+ return zero_stage, world_size, fp32_flat_groups
186
+
187
+
188
+ def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
189
+ """
190
+ Returns fp32 state_dict reconstructed from ds checkpoint
191
+
192
+ Args:
193
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
194
+
195
+ """
196
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
197
+
198
+ optim_files = get_optim_files(ds_checkpoint_dir)
199
+ zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
200
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
201
+
202
+ model_files = get_model_state_files(ds_checkpoint_dir)
203
+
204
+ zero_model_states = parse_model_states(model_files)
205
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
206
+
207
+ if zero_stage <= 2:
208
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
209
+ exclude_frozen_parameters)
210
+ elif zero_stage == 3:
211
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
212
+ exclude_frozen_parameters)
213
+
214
+
215
+ def _zero2_merge_frozen_params(state_dict, zero_model_states):
216
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
217
+ return
218
+
219
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
220
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
221
+
222
+ if debug:
223
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
224
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
225
+
226
+ wanted_params = len(frozen_param_shapes)
227
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
228
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
229
+ print(f'Frozen params: Have {avail_numel} numels to process.')
230
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
231
+
232
+ total_params = 0
233
+ total_numel = 0
234
+ for name, shape in frozen_param_shapes.items():
235
+ total_params += 1
236
+ unpartitioned_numel = shape.numel()
237
+ total_numel += unpartitioned_numel
238
+
239
+ state_dict[name] = frozen_param_fragments[name]
240
+
241
+ if debug:
242
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
243
+
244
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
245
+
246
+
247
+ def _has_callable(obj, fn):
248
+ attr = getattr(obj, fn, None)
249
+ return callable(attr)
250
+
251
+
252
+ def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
253
+ param_shapes = zero_model_states[0].param_shapes
254
+
255
+ # Reconstruction protocol:
256
+ #
257
+ # XXX: document this
258
+
259
+ if debug:
260
+ for i in range(world_size):
261
+ for j in range(len(fp32_flat_groups[0])):
262
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
263
+
264
+ # XXX: memory usage doubles here (zero2)
265
+ num_param_groups = len(fp32_flat_groups[0])
266
+ merged_single_partition_of_fp32_groups = []
267
+ for i in range(num_param_groups):
268
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
269
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
270
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
271
+ avail_numel = sum(
272
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
273
+
274
+ if debug:
275
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
276
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
277
+ # not asserting if there is a mismatch due to possible padding
278
+ print(f"Have {avail_numel} numels to process.")
279
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
280
+
281
+ # params
282
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
283
+ # out-of-core computing solution
284
+ total_numel = 0
285
+ total_params = 0
286
+ for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
287
+ offset = 0
288
+ avail_numel = full_single_fp32_vector.numel()
289
+ for name, shape in shapes.items():
290
+
291
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
292
+ total_numel += unpartitioned_numel
293
+ total_params += 1
294
+
295
+ if debug:
296
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
297
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
298
+ offset += unpartitioned_numel
299
+
300
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
301
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
302
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
303
+ # live optimizer object, so we are checking that the numbers are within the right range
304
+ align_to = 2 * world_size
305
+
306
+ def zero2_align(x):
307
+ return align_to * math.ceil(x / align_to)
308
+
309
+ if debug:
310
+ print(f"original offset={offset}, avail_numel={avail_numel}")
311
+
312
+ offset = zero2_align(offset)
313
+ avail_numel = zero2_align(avail_numel)
314
+
315
+ if debug:
316
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
317
+
318
+ # Sanity check
319
+ if offset != avail_numel:
320
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
321
+
322
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
323
+
324
+
325
+ def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
326
+ exclude_frozen_parameters):
327
+ state_dict = OrderedDict()
328
+
329
+ # buffers
330
+ buffers = zero_model_states[0].buffers
331
+ state_dict.update(buffers)
332
+ if debug:
333
+ print(f"added {len(buffers)} buffers")
334
+
335
+ if not exclude_frozen_parameters:
336
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
337
+
338
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
339
+
340
+ # recover shared parameters
341
+ for pair in zero_model_states[0].shared_params:
342
+ if pair[1] in state_dict:
343
+ state_dict[pair[0]] = state_dict[pair[1]]
344
+
345
+ return state_dict
346
+
347
+
348
+ def zero3_partitioned_param_info(unpartitioned_numel, world_size):
349
+ remainder = unpartitioned_numel % world_size
350
+ padding_numel = (world_size - remainder) if remainder else 0
351
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
352
+ return partitioned_numel, padding_numel
353
+
354
+
355
+ def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
356
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
357
+ return
358
+
359
+ if debug:
360
+ for i in range(world_size):
361
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
362
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
363
+
364
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
365
+ wanted_params = len(frozen_param_shapes)
366
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
367
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
368
+ print(f'Frozen params: Have {avail_numel} numels to process.')
369
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
370
+
371
+ total_params = 0
372
+ total_numel = 0
373
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
374
+ total_params += 1
375
+ unpartitioned_numel = shape.numel()
376
+ total_numel += unpartitioned_numel
377
+
378
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
379
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
380
+
381
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
382
+
383
+ if debug:
384
+ print(
385
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
386
+ )
387
+
388
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
389
+
390
+
391
+ class GatheredTensor:
392
+ """
393
+ A pseudo tensor that collects partitioned weights.
394
+ It is more memory efficient when there are multiple groups.
395
+ """
396
+
397
+ def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
398
+ self.flat_groups = flat_groups
399
+ self.flat_groups_offset = flat_groups_offset
400
+ self.offset = offset
401
+ self.partitioned_numel = partitioned_numel
402
+ self.shape = shape
403
+ self.dtype = self.flat_groups[0][0].dtype
404
+
405
+ def contiguous(self):
406
+ """
407
+ Merge partitioned weights from flat_groups into a single tensor.
408
+ """
409
+ end_idx = self.offset + self.partitioned_numel
410
+ world_size = len(self.flat_groups)
411
+ pad_flat_param_chunks = []
412
+
413
+ for rank_i in range(world_size):
414
+ # for each rank, we need to collect weights from related group/groups
415
+ flat_groups_at_rank_i = self.flat_groups[rank_i]
416
+ start_group_id = None
417
+ end_group_id = None
418
+ for group_id in range(len(self.flat_groups_offset)):
419
+ if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
420
+ start_group_id = group_id
421
+ if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
422
+ end_group_id = group_id
423
+ break
424
+ # collect weights from related group/groups
425
+ for group_id in range(start_group_id, end_group_id + 1):
426
+ flat_tensor = flat_groups_at_rank_i[group_id]
427
+ start_offset = self.offset - self.flat_groups_offset[group_id]
428
+ end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
429
+ pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
430
+
431
+ # collect weights from all ranks
432
+ pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
433
+ param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
434
+ return param
435
+
436
+
437
+ def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
438
+ param_shapes = zero_model_states[0].param_shapes
439
+ avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
440
+
441
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
442
+ # param, re-consolidating each param, while dealing with padding if any
443
+
444
+ # merge list of dicts, preserving order
445
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
446
+
447
+ if debug:
448
+ for i in range(world_size):
449
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
450
+
451
+ wanted_params = len(param_shapes)
452
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
453
+ # not asserting if there is a mismatch due to possible padding
454
+ avail_numel = fp32_flat_groups[0].numel() * world_size
455
+ print(f"Trainable params: Have {avail_numel} numels to process.")
456
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
457
+
458
+ # params
459
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
460
+ # out-of-core computing solution
461
+ offset = 0
462
+ total_numel = 0
463
+ total_params = 0
464
+ flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
465
+ for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
466
+ unpartitioned_numel = shape.numel()
467
+ total_numel += unpartitioned_numel
468
+ total_params += 1
469
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
470
+
471
+ if debug:
472
+ print(
473
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
474
+ )
475
+
476
+ # memory efficient tensor
477
+ tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
478
+ state_dict[name] = tensor
479
+ offset += partitioned_numel
480
+
481
+ offset *= world_size
482
+
483
+ # Sanity check
484
+ if offset != avail_numel:
485
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
486
+
487
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
488
+
489
+
490
+ def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
491
+ exclude_frozen_parameters):
492
+ state_dict = OrderedDict()
493
+
494
+ # buffers
495
+ buffers = zero_model_states[0].buffers
496
+ state_dict.update(buffers)
497
+ if debug:
498
+ print(f"added {len(buffers)} buffers")
499
+
500
+ if not exclude_frozen_parameters:
501
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
502
+
503
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
504
+
505
+ # recover shared parameters
506
+ for pair in zero_model_states[0].shared_params:
507
+ if pair[1] in state_dict:
508
+ state_dict[pair[0]] = state_dict[pair[1]]
509
+
510
+ return state_dict
511
+
512
+
513
+ def to_torch_tensor(state_dict, return_empty_tensor=False):
514
+ """
515
+ Convert state_dict of GatheredTensor to torch tensor
516
+ """
517
+ torch_state_dict = {}
518
+ converted_tensors = {}
519
+ for name, tensor in state_dict.items():
520
+ tensor_id = id(tensor)
521
+ if tensor_id in converted_tensors: # shared tensors
522
+ shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
523
+ torch_state_dict[name] = shared_tensor
524
+ else:
525
+ converted_tensors[tensor_id] = name
526
+ if return_empty_tensor:
527
+ torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
528
+ else:
529
+ torch_state_dict[name] = tensor.contiguous()
530
+ return torch_state_dict
531
+
532
+
533
+ def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
534
+ tag=None,
535
+ exclude_frozen_parameters=False,
536
+ lazy_mode=False):
537
+ """
538
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
539
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
540
+ via a model hub.
541
+
542
+ Args:
543
+ - ``checkpoint_dir``: path to the desired checkpoint folder
544
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
545
+ - ``exclude_frozen_parameters``: exclude frozen parameters
546
+ - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
547
+ Convert the pesduo tensor to torch tensor by ``.contiguous()``
548
+
549
+ Returns:
550
+ - pytorch ``state_dict``
551
+
552
+ A typical usage might be ::
553
+
554
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
555
+ # do the training and checkpoint saving
556
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
557
+ model = model.cpu() # move to cpu
558
+ model.load_state_dict(state_dict)
559
+ # submit to model hub or save the model to share with others
560
+
561
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
562
+ application. i.e. you will need to re-initialize the deepspeed engine, since
563
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
564
+
565
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
566
+
567
+ Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
568
+ You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
569
+ the checkpoint. Or you can load state_dict in lazy mode ::
570
+
571
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
572
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
573
+ for name, lazy_tensor in state_dict.item():
574
+ tensor = lazy_tensor.contiguous() # to cpu
575
+ print(name, tensor)
576
+ # del tensor to release memory if it no longer in use
577
+ """
578
+ if tag is None:
579
+ latest_path = os.path.join(checkpoint_dir, 'latest')
580
+ if os.path.isfile(latest_path):
581
+ with open(latest_path, 'r') as fd:
582
+ tag = fd.read().strip()
583
+ else:
584
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
585
+
586
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
587
+
588
+ if not os.path.isdir(ds_checkpoint_dir):
589
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
590
+
591
+ state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
592
+ if lazy_mode:
593
+ return state_dict
594
+ else:
595
+ return to_torch_tensor(state_dict)
596
+
597
+
598
+ def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
599
+ output_dir,
600
+ max_shard_size="5GB",
601
+ safe_serialization=False,
602
+ tag=None,
603
+ exclude_frozen_parameters=False):
604
+ """
605
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
606
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
607
+
608
+ Args:
609
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
610
+ - ``output_dir``: directory to the pytorch fp32 state_dict output files
611
+ - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
612
+ - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
613
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
614
+ - ``exclude_frozen_parameters``: exclude frozen parameters
615
+ """
616
+
617
+ # Dependency pre-check
618
+ if safe_serialization:
619
+ try:
620
+ from safetensors.torch import save_file
621
+ except ImportError:
622
+ print('If you want to use `safe_serialization`, please `pip install safetensors`')
623
+ raise
624
+ if max_shard_size is not None:
625
+ try:
626
+ from huggingface_hub import split_torch_state_dict_into_shards
627
+ except ImportError:
628
+ print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
629
+ raise
630
+
631
+ # Convert zero checkpoint to state_dict
632
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
633
+ tag,
634
+ exclude_frozen_parameters,
635
+ lazy_mode=True)
636
+
637
+ # Shard the model if it is too big.
638
+ weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
639
+ if max_shard_size is not None:
640
+ filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
641
+ # an memory-efficient approach for sharding
642
+ empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
643
+ state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
644
+ filename_pattern=filename_pattern,
645
+ max_shard_size=max_shard_size)
646
+ else:
647
+ from collections import namedtuple
648
+ StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
649
+ state_dict_split = StateDictSplit(is_sharded=False,
650
+ filename_to_tensors={weights_name: list(state_dict.keys())})
651
+
652
+ # Save the model by shard
653
+ os.makedirs(output_dir, exist_ok=True)
654
+ filename_to_tensors = state_dict_split.filename_to_tensors.items()
655
+ for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
656
+ shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
657
+ shard_state_dict = to_torch_tensor(shard_state_dict)
658
+ output_path = os.path.join(output_dir, shard_file)
659
+ if safe_serialization:
660
+ save_file(shard_state_dict, output_path, metadata={"format": "pt"})
661
+ else:
662
+ torch.save(shard_state_dict, output_path)
663
+ # release the memory of current shard
664
+ for tensor_name in list(shard_state_dict.keys()):
665
+ del state_dict[tensor_name]
666
+ del shard_state_dict[tensor_name]
667
+ del shard_state_dict
668
+ gc.collect()
669
+
670
+ # Save index if sharded
671
+ if state_dict_split.is_sharded:
672
+ index = {
673
+ "metadata": state_dict_split.metadata,
674
+ "weight_map": state_dict_split.tensor_to_filename,
675
+ }
676
+ save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
677
+ save_index_file = os.path.join(output_dir, save_index_file)
678
+ with open(save_index_file, "w", encoding="utf-8") as f:
679
+ content = json.dumps(index, indent=2, sort_keys=True) + "\n"
680
+ f.write(content)
681
+
682
+
683
+ def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
684
+ """
685
+ 1. Put the provided model to cpu
686
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
687
+ 3. Load it into the provided model
688
+
689
+ Args:
690
+ - ``model``: the model object to update
691
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
692
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
693
+
694
+ Returns:
695
+ - ``model`: modified model
696
+
697
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
698
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
699
+ conveniently placed for you in the checkpoint folder.
700
+
701
+ A typical usage might be ::
702
+
703
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
704
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
705
+ # submit to model hub or save the model to share with others
706
+
707
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
708
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
709
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
710
+
711
+ """
712
+ logger.info(f"Extracting fp32 weights")
713
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
714
+
715
+ logger.info(f"Overwriting model with fp32 weights")
716
+ model = model.cpu()
717
+ model.load_state_dict(state_dict, strict=False)
718
+
719
+ return model
720
+
721
+
722
+ if __name__ == "__main__":
723
+ parser = argparse.ArgumentParser()
724
+ parser.add_argument("checkpoint_dir",
725
+ type=str,
726
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
727
+ parser.add_argument("output_dir",
728
+ type=str,
729
+ help="directory to the pytorch fp32 state_dict output files"
730
+ "(e.g. path/checkpoint-12-output/)")
731
+ parser.add_argument(
732
+ "--max_shard_size",
733
+ type=str,
734
+ default="5GB",
735
+ help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
736
+ "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
737
+ "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
738
+ "without CPU OOM issues.")
739
+ parser.add_argument(
740
+ "--safe_serialization",
741
+ default=False,
742
+ action='store_true',
743
+ help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
744
+ parser.add_argument("-t",
745
+ "--tag",
746
+ type=str,
747
+ default=None,
748
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
749
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
750
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
751
+ args = parser.parse_args()
752
+
753
+ debug = args.debug
754
+
755
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
756
+ args.output_dir,
757
+ max_shard_size=args.max_shard_size,
758
+ safe_serialization=args.safe_serialization,
759
+ tag=args.tag,
760
+ exclude_frozen_parameters=args.exclude_frozen_parameters)
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/added_tokens.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "</tool_call>": 151658,
3
+ "<tool_call>": 151657,
4
+ "<|box_end|>": 151649,
5
+ "<|box_start|>": 151648,
6
+ "<|endoftext|>": 151643,
7
+ "<|file_sep|>": 151664,
8
+ "<|fim_middle|>": 151660,
9
+ "<|fim_pad|>": 151662,
10
+ "<|fim_prefix|>": 151659,
11
+ "<|fim_suffix|>": 151661,
12
+ "<|im_end|>": 151645,
13
+ "<|im_start|>": 151644,
14
+ "<|image_pad|>": 151655,
15
+ "<|object_ref_end|>": 151647,
16
+ "<|object_ref_start|>": 151646,
17
+ "<|quad_end|>": 151651,
18
+ "<|quad_start|>": 151650,
19
+ "<|repo_name|>": 151663,
20
+ "<|video_pad|>": 151656,
21
+ "<|vision_end|>": 151653,
22
+ "<|vision_pad|>": 151654,
23
+ "<|vision_start|>": 151652
24
+ }
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/args.json ADDED
@@ -0,0 +1,382 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "output_dir": "/group/40143/hongzhuyi/ms-swift/output/v1-20250912-013615",
3
+ "overwrite_output_dir": false,
4
+ "do_train": false,
5
+ "do_eval": false,
6
+ "do_predict": false,
7
+ "eval_strategy": "epoch",
8
+ "prediction_loss_only": false,
9
+ "per_device_train_batch_size": 2,
10
+ "per_device_eval_batch_size": 1,
11
+ "per_gpu_train_batch_size": null,
12
+ "per_gpu_eval_batch_size": null,
13
+ "gradient_accumulation_steps": 4,
14
+ "eval_accumulation_steps": null,
15
+ "eval_delay": 0,
16
+ "torch_empty_cache_steps": null,
17
+ "learning_rate": 1e-06,
18
+ "weight_decay": 0.1,
19
+ "adam_beta1": 0.9,
20
+ "adam_beta2": 0.95,
21
+ "adam_epsilon": 1e-08,
22
+ "max_grad_norm": 1.0,
23
+ "num_train_epochs": 2.0,
24
+ "max_steps": -1,
25
+ "lr_scheduler_type": "cosine",
26
+ "lr_scheduler_kwargs": null,
27
+ "warmup_ratio": 0.05,
28
+ "warmup_steps": 0,
29
+ "log_level": "passive",
30
+ "log_level_replica": "warning",
31
+ "log_on_each_node": true,
32
+ "logging_dir": "/group/40143/hongzhuyi/ms-swift/output/v1-20250912-013615/runs",
33
+ "logging_strategy": "steps",
34
+ "logging_first_step": true,
35
+ "logging_steps": 1,
36
+ "logging_nan_inf_filter": true,
37
+ "save_strategy": "epoch",
38
+ "save_steps": 500,
39
+ "save_total_limit": null,
40
+ "save_safetensors": true,
41
+ "save_on_each_node": false,
42
+ "save_only_model": false,
43
+ "restore_callback_states_from_checkpoint": false,
44
+ "no_cuda": false,
45
+ "use_cpu": false,
46
+ "use_mps_device": false,
47
+ "seed": 42,
48
+ "data_seed": 42,
49
+ "jit_mode_eval": false,
50
+ "use_ipex": false,
51
+ "bf16": true,
52
+ "fp16": false,
53
+ "fp16_opt_level": "O1",
54
+ "half_precision_backend": "auto",
55
+ "bf16_full_eval": false,
56
+ "fp16_full_eval": false,
57
+ "tf32": null,
58
+ "local_rank": 0,
59
+ "ddp_backend": null,
60
+ "tpu_num_cores": null,
61
+ "tpu_metrics_debug": false,
62
+ "debug": null,
63
+ "dataloader_drop_last": false,
64
+ "eval_steps": 2000.0,
65
+ "dataloader_num_workers": 48,
66
+ "dataloader_prefetch_factor": null,
67
+ "past_index": -1,
68
+ "run_name": "/group/40143/hongzhuyi/ms-swift/output/v1-20250912-013615",
69
+ "disable_tqdm": null,
70
+ "remove_unused_columns": true,
71
+ "label_names": null,
72
+ "load_best_model_at_end": false,
73
+ "metric_for_best_model": "loss",
74
+ "greater_is_better": false,
75
+ "ignore_data_skip": false,
76
+ "fsdp": "",
77
+ "fsdp_min_num_params": 0,
78
+ "fsdp_config": null,
79
+ "fsdp_transformer_layer_cls_to_wrap": null,
80
+ "accelerator_config": {
81
+ "dispatch_batches": false
82
+ },
83
+ "deepspeed": {
84
+ "fp16": {
85
+ "enabled": "auto",
86
+ "loss_scale": 0,
87
+ "loss_scale_window": 1000,
88
+ "initial_scale_power": 16,
89
+ "hysteresis": 2,
90
+ "min_loss_scale": 1
91
+ },
92
+ "bf16": {
93
+ "enabled": "auto"
94
+ },
95
+ "zero_optimization": {
96
+ "stage": 3,
97
+ "offload_optimizer": {
98
+ "device": "none",
99
+ "pin_memory": true
100
+ },
101
+ "offload_param": {
102
+ "device": "none",
103
+ "pin_memory": true
104
+ },
105
+ "overlap_comm": false,
106
+ "contiguous_gradients": true,
107
+ "sub_group_size": 1000000000.0,
108
+ "reduce_bucket_size": "auto",
109
+ "zero_quantized_weights": false,
110
+ "zero_quantized_gradients": false,
111
+ "stage3_prefetch_bucket_size": "auto",
112
+ "stage3_param_persistence_threshold": "auto",
113
+ "stage3_max_live_parameters": 1000000000.0,
114
+ "stage3_max_reuse_distance": 1000000000.0,
115
+ "stage3_gather_16bit_weights_on_model_save": true
116
+ },
117
+ "gradient_accumulation_steps": "auto",
118
+ "gradient_clipping": "auto",
119
+ "steps_per_print": 2000,
120
+ "train_batch_size": "auto",
121
+ "train_micro_batch_size_per_gpu": "auto",
122
+ "wall_clock_breakdown": false
123
+ },
124
+ "label_smoothing_factor": 0.0,
125
+ "optim": "adamw_torch_fused",
126
+ "optim_args": null,
127
+ "adafactor": false,
128
+ "group_by_length": false,
129
+ "length_column_name": "length",
130
+ "report_to": [
131
+ "tensorboard"
132
+ ],
133
+ "ddp_find_unused_parameters": null,
134
+ "ddp_bucket_cap_mb": null,
135
+ "ddp_broadcast_buffers": null,
136
+ "dataloader_pin_memory": true,
137
+ "dataloader_persistent_workers": false,
138
+ "skip_memory_metrics": true,
139
+ "use_legacy_prediction_loop": false,
140
+ "push_to_hub": false,
141
+ "resume_from_checkpoint": null,
142
+ "hub_model_id": null,
143
+ "hub_strategy": "every_save",
144
+ "hub_token": null,
145
+ "hub_private_repo": null,
146
+ "hub_always_push": false,
147
+ "hub_revision": null,
148
+ "gradient_checkpointing": true,
149
+ "gradient_checkpointing_kwargs": null,
150
+ "include_inputs_for_metrics": false,
151
+ "include_for_metrics": [],
152
+ "eval_do_concat_batches": true,
153
+ "fp16_backend": "auto",
154
+ "push_to_hub_model_id": null,
155
+ "push_to_hub_organization": null,
156
+ "push_to_hub_token": null,
157
+ "mp_parameters": "",
158
+ "auto_find_batch_size": false,
159
+ "full_determinism": false,
160
+ "torchdynamo": null,
161
+ "ray_scope": "last",
162
+ "ddp_timeout": 18000000,
163
+ "torch_compile": false,
164
+ "torch_compile_backend": null,
165
+ "torch_compile_mode": null,
166
+ "include_tokens_per_second": false,
167
+ "include_num_input_tokens_seen": false,
168
+ "neftune_noise_alpha": null,
169
+ "optim_target_modules": null,
170
+ "batch_eval_metrics": false,
171
+ "eval_on_start": false,
172
+ "use_liger_kernel": false,
173
+ "liger_kernel_config": null,
174
+ "eval_use_gather_object": false,
175
+ "average_tokens_across_devices": true,
176
+ "sortish_sampler": false,
177
+ "predict_with_generate": false,
178
+ "generation_max_length": null,
179
+ "generation_num_beams": null,
180
+ "generation_config": null,
181
+ "tuner_backend": "peft",
182
+ "vit_gradient_checkpointing": null,
183
+ "router_aux_loss_coef": 0.0,
184
+ "enable_dft_loss": false,
185
+ "check_model": true,
186
+ "acc_strategy": "token",
187
+ "train_dataloader_shuffle": true,
188
+ "max_epochs": null,
189
+ "aligner_lr": null,
190
+ "vit_lr": null,
191
+ "use_logits_to_keep": null,
192
+ "channels": null,
193
+ "ds3_gather_for_generation": true,
194
+ "resume_only_model": false,
195
+ "optimizer": null,
196
+ "loss_type": null,
197
+ "metric": null,
198
+ "eval_use_evalscope": false,
199
+ "eval_dataset": [],
200
+ "eval_dataset_args": null,
201
+ "eval_limit": null,
202
+ "eval_generation_config": null,
203
+ "extra_eval_args": null,
204
+ "use_flash_ckpt": false,
205
+ "model": "Qwen/Qwen2.5-7B-Instruct",
206
+ "model_type": "qwen2_5",
207
+ "model_revision": null,
208
+ "task_type": "causal_lm",
209
+ "torch_dtype": "bfloat16",
210
+ "attn_impl": null,
211
+ "new_special_tokens": [],
212
+ "num_labels": null,
213
+ "problem_type": null,
214
+ "rope_scaling": null,
215
+ "device_map": null,
216
+ "max_memory": {},
217
+ "max_model_len": null,
218
+ "local_repo_path": null,
219
+ "init_strategy": null,
220
+ "template": "qwen2_5",
221
+ "system": null,
222
+ "max_length": 16240,
223
+ "truncation_strategy": "delete",
224
+ "max_pixels": null,
225
+ "agent_template": null,
226
+ "norm_bbox": null,
227
+ "use_chat_template": true,
228
+ "padding_free": false,
229
+ "padding_side": "right",
230
+ "loss_scale": "default",
231
+ "sequence_parallel_size": 1,
232
+ "response_prefix": null,
233
+ "template_backend": "swift",
234
+ "dataset": [
235
+ "./corr_hotpot_2083q_context_swift.jsonl",
236
+ "./corr_hotpot_new1369q_context_swift.jsonl",
237
+ "./corr_nq_2225q_context_swift.jsonl"
238
+ ],
239
+ "val_dataset": [],
240
+ "split_dataset_ratio": 0.001,
241
+ "dataset_num_proc": 100,
242
+ "load_from_cache_file": true,
243
+ "dataset_shuffle": true,
244
+ "val_dataset_shuffle": false,
245
+ "streaming": false,
246
+ "interleave_prob": null,
247
+ "stopping_strategy": "first_exhausted",
248
+ "shuffle_buffer_size": 1000,
249
+ "download_mode": "reuse_dataset_if_exists",
250
+ "columns": {},
251
+ "strict": false,
252
+ "model_name": null,
253
+ "model_author": null,
254
+ "custom_dataset_info": [],
255
+ "quant_method": null,
256
+ "quant_bits": null,
257
+ "hqq_axis": null,
258
+ "bnb_4bit_compute_dtype": "bfloat16",
259
+ "bnb_4bit_quant_type": "nf4",
260
+ "bnb_4bit_use_double_quant": true,
261
+ "bnb_4bit_quant_storage": null,
262
+ "max_new_tokens": 64,
263
+ "temperature": 0.0,
264
+ "top_k": null,
265
+ "top_p": null,
266
+ "repetition_penalty": null,
267
+ "num_beams": 1,
268
+ "stream": false,
269
+ "stop_words": [],
270
+ "logprobs": false,
271
+ "top_logprobs": null,
272
+ "ckpt_dir": null,
273
+ "lora_modules": [],
274
+ "train_type": "full",
275
+ "adapters": [],
276
+ "external_plugins": [],
277
+ "model_kwargs": {},
278
+ "load_args": false,
279
+ "load_data_args": false,
280
+ "packing": false,
281
+ "packing_length": null,
282
+ "lazy_tokenize": false,
283
+ "cached_dataset": [],
284
+ "custom_register_path": [],
285
+ "use_hf": false,
286
+ "ignore_args_error": false,
287
+ "use_swift_lora": false,
288
+ "freeze_parameters": [],
289
+ "freeze_parameters_regex": null,
290
+ "freeze_parameters_ratio": 0.0,
291
+ "trainable_parameters": [],
292
+ "trainable_parameters_regex": null,
293
+ "freeze_llm": false,
294
+ "freeze_vit": true,
295
+ "freeze_aligner": false,
296
+ "target_modules": [
297
+ "all-linear"
298
+ ],
299
+ "target_regex": null,
300
+ "modules_to_save": [],
301
+ "lora_rank": 8,
302
+ "lora_alpha": 32,
303
+ "lora_dropout": 0.05,
304
+ "lora_bias": "none",
305
+ "lora_dtype": null,
306
+ "lorap_lr_ratio": null,
307
+ "use_rslora": false,
308
+ "use_dora": false,
309
+ "lora_ga_batch_size": 2,
310
+ "lora_ga_iters": 2,
311
+ "lora_ga_max_length": 1024,
312
+ "lora_ga_direction": "ArB2r",
313
+ "lora_ga_scale": "stable",
314
+ "lora_ga_stable_gamma": 16,
315
+ "init_weights": true,
316
+ "fourier_n_frequency": 2000,
317
+ "fourier_scaling": 300.0,
318
+ "boft_block_size": 4,
319
+ "boft_block_num": 0,
320
+ "boft_n_butterfly_factor": 1,
321
+ "boft_dropout": 0.0,
322
+ "vera_rank": 256,
323
+ "vera_projection_prng_key": 0,
324
+ "vera_dropout": 0.0,
325
+ "vera_d_initial": 0.1,
326
+ "adapter_act": "gelu",
327
+ "adapter_length": 128,
328
+ "use_galore": false,
329
+ "galore_target_modules": null,
330
+ "galore_rank": 128,
331
+ "galore_update_proj_gap": 50,
332
+ "galore_scale": 1.0,
333
+ "galore_proj_type": "std",
334
+ "galore_optim_per_parameter": false,
335
+ "galore_with_embedding": false,
336
+ "galore_quantization": false,
337
+ "galore_proj_quant": false,
338
+ "galore_proj_bits": 4,
339
+ "galore_proj_group_size": 256,
340
+ "galore_cos_threshold": 0.4,
341
+ "galore_gamma_proj": 2,
342
+ "galore_queue_size": 5,
343
+ "adalora_target_r": 8,
344
+ "adalora_init_r": 12,
345
+ "adalora_tinit": 0,
346
+ "adalora_tfinal": 0,
347
+ "adalora_deltaT": 1,
348
+ "adalora_beta1": 0.85,
349
+ "adalora_beta2": 0.85,
350
+ "adalora_orth_reg_weight": 0.5,
351
+ "llamapro_num_new_blocks": 4,
352
+ "llamapro_num_groups": null,
353
+ "lisa_activated_layers": 0,
354
+ "lisa_step_interval": 20,
355
+ "reft_layer_key": null,
356
+ "reft_layers": null,
357
+ "reft_rank": 4,
358
+ "reft_intervention_type": "LoreftIntervention",
359
+ "reft_args": null,
360
+ "swanlab_token": null,
361
+ "swanlab_project": null,
362
+ "swanlab_workspace": null,
363
+ "swanlab_exp_name": null,
364
+ "swanlab_lark_webhook_url": null,
365
+ "swanlab_lark_secret": null,
366
+ "swanlab_mode": "cloud",
367
+ "add_version": true,
368
+ "create_checkpoint_symlink": false,
369
+ "zero_hpz_partition_size": null,
370
+ "deepspeed_autotp_size": null,
371
+ "early_stop_interval": null,
372
+ "rank": 0,
373
+ "global_world_size": 8,
374
+ "local_world_size": 8,
375
+ "model_suffix": "Qwen2.5-7B-Instruct",
376
+ "model_info": "ModelInfo(model_type='qwen2_5', model_dir='/usr/local/app/.cache/modelscope/hub/models/Qwen/Qwen2___5-7B-Instruct', torch_dtype=torch.bfloat16, max_model_len=32768, quant_method=None, quant_bits=None, rope_scaling=None, is_moe_model=False, config=None, task_type='causal_lm', num_labels=None)",
377
+ "model_meta": "ModelMeta(model_type='qwen2_5', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct', hf_model_id='Qwen/Qwen2.5-3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct', hf_model_id='Qwen/Qwen2.5-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct', hf_model_id='Qwen/Qwen2.5-14B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct', hf_model_id='Qwen/Qwen2.5-32B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct', hf_model_id='Qwen/Qwen2.5-72B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B', hf_model_id='Qwen/Qwen2.5-0.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B', hf_model_id='Qwen/Qwen2.5-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B', hf_model_id='Qwen/Qwen2.5-3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B', hf_model_id='Qwen/Qwen2.5-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B', hf_model_id='Qwen/Qwen2.5-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B', hf_model_id='Qwen/Qwen2.5-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B', hf_model_id='Qwen/Qwen2.5-72B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-3B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-7B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-14B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-32B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-72B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B', hf_model_id='Qwen/Qwen2.5-Coder-0.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B', hf_model_id='Qwen/Qwen2.5-Coder-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B', hf_model_id='Qwen/Qwen2.5-Coder-3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B', hf_model_id='Qwen/Qwen2.5-Coder-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B', hf_model_id='Qwen/Qwen2.5-Coder-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B', hf_model_id='Qwen/Qwen2.5-Coder-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=['coding']), ModelGroup(models=[Model(ms_model_id='moonshotai/Kimi-Dev-72B', hf_model_id='moonshotai/Kimi-Dev-72B', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[])], template='qwen2_5', get_function=<function get_model_tokenizer_with_flash_attn at 0x7f4f5af767a0>, model_arch=ModelKeys(arch_name='llama', embedding='model.embed_tokens', module_list='model.layers', lm_head='lm_head', q_proj='model.layers.{}.self_attn.q_proj', k_proj='model.layers.{}.self_attn.k_proj', v_proj='model.layers.{}.self_attn.v_proj', o_proj='model.layers.{}.self_attn.o_proj', attention='model.layers.{}.self_attn', mlp='model.layers.{}.mlp', down_proj='model.layers.{}.mlp.down_proj', qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None), architectures=['Qwen2ForCausalLM'], additional_saved_files=[], torch_dtype=None, is_multimodal=False, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.37'], tags=[])",
378
+ "model_dir": "/usr/local/app/.cache/modelscope/hub/models/Qwen/Qwen2___5-7B-Instruct",
379
+ "hub": "<class 'swift.hub.hub.MSHub'>",
380
+ "evaluation_strategy": "epoch",
381
+ "training_args": "Seq2SeqTrainingArguments(output_dir='/group/40143/hongzhuyi/ms-swift/output/v1-20250912-013615', overwrite_output_dir=False, do_train=False, do_eval=True, do_predict=False, eval_strategy=<IntervalStrategy.EPOCH: 'epoch'>, prediction_loss_only=False, per_device_train_batch_size=2, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=4, eval_accumulation_steps=None, eval_delay=0, torch_empty_cache_steps=None, learning_rate=1e-06, weight_decay=0.1, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=2.0, max_steps=-1, lr_scheduler_type=<SchedulerType.COSINE: 'cosine'>, lr_scheduler_kwargs=None, warmup_ratio=0.05, warmup_steps=0, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='/group/40143/hongzhuyi/ms-swift/output/v1-20250912-013615/runs', logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_first_step=True, logging_steps=1, logging_nan_inf_filter=True, save_strategy=<SaveStrategy.EPOCH: 'epoch'>, save_steps=500, save_total_limit=None, save_safetensors=True, save_on_each_node=False, save_only_model=False, restore_callback_states_from_checkpoint=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=42, data_seed=42, jit_mode_eval=False, use_ipex=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=2000.0, dataloader_num_workers=48, dataloader_prefetch_factor=10, past_index=-1, run_name='/group/40143/hongzhuyi/ms-swift/output/v1-20250912-013615', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model='loss', greater_is_better=False, ignore_data_skip=False, fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), deepspeed={'fp16': {'enabled': 'auto', 'loss_scale': 0, 'loss_scale_window': 1000, 'initial_scale_power': 16, 'hysteresis': 2, 'min_loss_scale': 1}, 'bf16': {'enabled': 'auto'}, 'zero_optimization': {'stage': 3, 'offload_optimizer': {'device': 'none', 'pin_memory': True}, 'offload_param': {'device': 'none', 'pin_memory': True}, 'overlap_comm': False, 'contiguous_gradients': True, 'sub_group_size': 1000000000.0, 'reduce_bucket_size': 'auto', 'zero_quantized_weights': False, 'zero_quantized_gradients': False, 'stage3_prefetch_bucket_size': 'auto', 'stage3_param_persistence_threshold': 'auto', 'stage3_max_live_parameters': 1000000000.0, 'stage3_max_reuse_distance': 1000000000.0, 'stage3_gather_16bit_weights_on_model_save': True}, 'gradient_accumulation_steps': 'auto', 'gradient_clipping': 'auto', 'steps_per_print': 2000, 'train_batch_size': 'auto', 'train_micro_batch_size_per_gpu': 'auto', 'wall_clock_breakdown': False}, label_smoothing_factor=0.0, optim=<OptimizerNames.ADAMW_TORCH_FUSED: 'adamw_torch_fused'>, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], ddp_find_unused_parameters=None, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, dataloader_persistent_workers=False, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_token=None, hub_private_repo=None, hub_always_push=False, hub_revision=None, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, include_inputs_for_metrics=False, include_for_metrics=[], eval_do_concat_batches=True, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=18000000, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, include_tokens_per_second=None, include_num_input_tokens_seen=None, neftune_noise_alpha=None, optim_target_modules=None, batch_eval_metrics=False, eval_on_start=False, use_liger_kernel=False, liger_kernel_config=None, eval_use_gather_object=False, average_tokens_across_devices=None, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=None, tuner_backend='peft', vit_gradient_checkpointing=True, router_aux_loss_coef=0.0, enable_dft_loss=False, check_model=True, acc_strategy='token', train_dataloader_shuffle=True, max_epochs=None, aligner_lr=None, vit_lr=None, use_logits_to_keep=None, channels=None, ds3_gather_for_generation=True, resume_only_model=False, optimizer=None, loss_type=None, metric=None, eval_use_evalscope=False, eval_dataset=[], eval_dataset_args=None, eval_limit=None, eval_generation_config=None, extra_eval_args=None, use_flash_ckpt=False, sft_alpha=0, train_type='full', local_repo_path=None, galore_config=None)"
382
+ }
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/chat_template.jinja ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {%- if tools %}
2
+ {{- '<|im_start|>system\n' }}
3
+ {%- if messages[0]['role'] == 'system' %}
4
+ {{- messages[0]['content'] }}
5
+ {%- else %}
6
+ {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
7
+ {%- endif %}
8
+ {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
9
+ {%- for tool in tools %}
10
+ {{- "\n" }}
11
+ {{- tool | tojson }}
12
+ {%- endfor %}
13
+ {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
14
+ {%- else %}
15
+ {%- if messages[0]['role'] == 'system' %}
16
+ {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
17
+ {%- else %}
18
+ {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
19
+ {%- endif %}
20
+ {%- endif %}
21
+ {%- for message in messages %}
22
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
23
+ {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
24
+ {%- elif message.role == "assistant" %}
25
+ {{- '<|im_start|>' + message.role }}
26
+ {%- if message.content %}
27
+ {{- '\n' + message.content }}
28
+ {%- endif %}
29
+ {%- for tool_call in message.tool_calls %}
30
+ {%- if tool_call.function is defined %}
31
+ {%- set tool_call = tool_call.function %}
32
+ {%- endif %}
33
+ {{- '\n<tool_call>\n{"name": "' }}
34
+ {{- tool_call.name }}
35
+ {{- '", "arguments": ' }}
36
+ {{- tool_call.arguments | tojson }}
37
+ {{- '}\n</tool_call>' }}
38
+ {%- endfor %}
39
+ {{- '<|im_end|>\n' }}
40
+ {%- elif message.role == "tool" %}
41
+ {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
42
+ {{- '<|im_start|>user' }}
43
+ {%- endif %}
44
+ {{- '\n<tool_response>\n' }}
45
+ {{- message.content }}
46
+ {{- '\n</tool_response>' }}
47
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
48
+ {{- '<|im_end|>\n' }}
49
+ {%- endif %}
50
+ {%- endif %}
51
+ {%- endfor %}
52
+ {%- if add_generation_prompt %}
53
+ {{- '<|im_start|>assistant\n' }}
54
+ {%- endif %}
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/config.json ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen2ForCausalLM"
4
+ ],
5
+ "attention_dropout": 0.0,
6
+ "bos_token_id": 151643,
7
+ "eos_token_id": 151645,
8
+ "hidden_act": "silu",
9
+ "hidden_size": 3584,
10
+ "initializer_range": 0.02,
11
+ "intermediate_size": 18944,
12
+ "layer_types": [
13
+ "full_attention",
14
+ "full_attention",
15
+ "full_attention",
16
+ "full_attention",
17
+ "full_attention",
18
+ "full_attention",
19
+ "full_attention",
20
+ "full_attention",
21
+ "full_attention",
22
+ "full_attention",
23
+ "full_attention",
24
+ "full_attention",
25
+ "full_attention",
26
+ "full_attention",
27
+ "full_attention",
28
+ "full_attention",
29
+ "full_attention",
30
+ "full_attention",
31
+ "full_attention",
32
+ "full_attention",
33
+ "full_attention",
34
+ "full_attention",
35
+ "full_attention",
36
+ "full_attention",
37
+ "full_attention",
38
+ "full_attention",
39
+ "full_attention",
40
+ "full_attention"
41
+ ],
42
+ "max_position_embeddings": 32768,
43
+ "max_window_layers": 28,
44
+ "model_type": "qwen2",
45
+ "num_attention_heads": 28,
46
+ "num_hidden_layers": 28,
47
+ "num_key_value_heads": 4,
48
+ "pad_token_id": 151643,
49
+ "rms_norm_eps": 1e-06,
50
+ "rope_scaling": null,
51
+ "rope_theta": 1000000.0,
52
+ "sliding_window": null,
53
+ "tie_word_embeddings": false,
54
+ "torch_dtype": "bfloat16",
55
+ "transformers_version": "4.55.4",
56
+ "use_cache": false,
57
+ "use_sliding_window": false,
58
+ "vocab_size": 152064
59
+ }
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/generation_config.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 151643,
3
+ "do_sample": true,
4
+ "eos_token_id": [
5
+ 151645,
6
+ 151643
7
+ ],
8
+ "pad_token_id": 151643,
9
+ "repetition_penalty": 1.05,
10
+ "temperature": 0.7,
11
+ "top_k": 20,
12
+ "top_p": 0.8,
13
+ "transformers_version": "4.55.4"
14
+ }
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/latest ADDED
@@ -0,0 +1 @@
 
 
1
+ global_step86
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/model-00001-of-00004.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fe1bc7bd54865ba0d7df2df1a4e5c3a452c4f110a9679e523d591ea7def0b388
3
+ size 4877660776
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/model-00002-of-00004.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:24f79207ab6361f18734bef1bb7f3e38be328fd7f3bb28bbe93b4976137aa445
3
+ size 4932751008
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/model-00003-of-00004.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dd1f1f8757d5eed99a2f07a76d77bbb22889d39ad5292c4ad93662e9a4a26a3f
3
+ size 4330865200
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/model-00004-of-00004.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:682ad0a3a8e8060d9bd60134ba68b2ee2c5d5d8b26f0a3e79a8f5124795493b4
3
+ size 1089994880
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/model.safetensors.index.json ADDED
@@ -0,0 +1,347 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "metadata": {
3
+ "total_parameters": 333312,
4
+ "total_size": 15231233024
5
+ },
6
+ "weight_map": {
7
+ "lm_head.weight": "model-00004-of-00004.safetensors",
8
+ "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
9
+ "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
10
+ "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
11
+ "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
12
+ "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
13
+ "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
14
+ "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
15
+ "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
16
+ "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
17
+ "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
18
+ "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
19
+ "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
20
+ "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
21
+ "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
22
+ "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
23
+ "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
24
+ "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
25
+ "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
26
+ "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
27
+ "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
28
+ "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
29
+ "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
30
+ "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
31
+ "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
32
+ "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
33
+ "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
34
+ "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
35
+ "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
36
+ "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
37
+ "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
38
+ "model.layers.10.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
39
+ "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
40
+ "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
41
+ "model.layers.10.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
42
+ "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
43
+ "model.layers.10.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
44
+ "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
45
+ "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
46
+ "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
47
+ "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
48
+ "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
49
+ "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
50
+ "model.layers.11.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
51
+ "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
52
+ "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
53
+ "model.layers.11.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
54
+ "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
55
+ "model.layers.11.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
56
+ "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
57
+ "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
58
+ "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
59
+ "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
60
+ "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
61
+ "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
62
+ "model.layers.12.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
63
+ "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
64
+ "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
65
+ "model.layers.12.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
66
+ "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
67
+ "model.layers.12.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
68
+ "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
69
+ "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
70
+ "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
71
+ "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
72
+ "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
73
+ "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
74
+ "model.layers.13.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
75
+ "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
76
+ "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
77
+ "model.layers.13.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
78
+ "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
79
+ "model.layers.13.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
80
+ "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
81
+ "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
82
+ "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
83
+ "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
84
+ "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
85
+ "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
86
+ "model.layers.14.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
87
+ "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
88
+ "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
89
+ "model.layers.14.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
90
+ "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
91
+ "model.layers.14.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
92
+ "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
93
+ "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
94
+ "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
95
+ "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
96
+ "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
97
+ "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
98
+ "model.layers.15.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
99
+ "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
100
+ "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
101
+ "model.layers.15.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
102
+ "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
103
+ "model.layers.15.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
104
+ "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
105
+ "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
106
+ "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
107
+ "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
108
+ "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
109
+ "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
110
+ "model.layers.16.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
111
+ "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
112
+ "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
113
+ "model.layers.16.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
114
+ "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
115
+ "model.layers.16.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
116
+ "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
117
+ "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
118
+ "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
119
+ "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
120
+ "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
121
+ "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
122
+ "model.layers.17.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
123
+ "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
124
+ "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
125
+ "model.layers.17.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
126
+ "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
127
+ "model.layers.17.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
128
+ "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
129
+ "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
130
+ "model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
131
+ "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
132
+ "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
133
+ "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
134
+ "model.layers.18.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
135
+ "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
136
+ "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
137
+ "model.layers.18.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
138
+ "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
139
+ "model.layers.18.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
140
+ "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
141
+ "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
142
+ "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
143
+ "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
144
+ "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
145
+ "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
146
+ "model.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
147
+ "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
148
+ "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
149
+ "model.layers.19.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
150
+ "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
151
+ "model.layers.19.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
152
+ "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
153
+ "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
154
+ "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
155
+ "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
156
+ "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
157
+ "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
158
+ "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
159
+ "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
160
+ "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
161
+ "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
162
+ "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
163
+ "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
164
+ "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
165
+ "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
166
+ "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
167
+ "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
168
+ "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
169
+ "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
170
+ "model.layers.20.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
171
+ "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
172
+ "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
173
+ "model.layers.20.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
174
+ "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
175
+ "model.layers.20.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
176
+ "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
177
+ "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
178
+ "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
179
+ "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
180
+ "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
181
+ "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
182
+ "model.layers.21.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
183
+ "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
184
+ "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
185
+ "model.layers.21.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
186
+ "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
187
+ "model.layers.21.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
188
+ "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
189
+ "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
190
+ "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
191
+ "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
192
+ "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
193
+ "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
194
+ "model.layers.22.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
195
+ "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
196
+ "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
197
+ "model.layers.22.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
198
+ "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
199
+ "model.layers.22.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
200
+ "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
201
+ "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
202
+ "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
203
+ "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
204
+ "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
205
+ "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
206
+ "model.layers.23.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
207
+ "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
208
+ "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
209
+ "model.layers.23.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
210
+ "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
211
+ "model.layers.23.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
212
+ "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
213
+ "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
214
+ "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
215
+ "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
216
+ "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
217
+ "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
218
+ "model.layers.24.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
219
+ "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
220
+ "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
221
+ "model.layers.24.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
222
+ "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
223
+ "model.layers.24.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
224
+ "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
225
+ "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
226
+ "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
227
+ "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
228
+ "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
229
+ "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
230
+ "model.layers.25.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
231
+ "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
232
+ "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
233
+ "model.layers.25.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
234
+ "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
235
+ "model.layers.25.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
236
+ "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
237
+ "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
238
+ "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
239
+ "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
240
+ "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
241
+ "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
242
+ "model.layers.26.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
243
+ "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
244
+ "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
245
+ "model.layers.26.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
246
+ "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
247
+ "model.layers.26.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
248
+ "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
249
+ "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
250
+ "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
251
+ "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
252
+ "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
253
+ "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
254
+ "model.layers.27.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
255
+ "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
256
+ "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
257
+ "model.layers.27.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
258
+ "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
259
+ "model.layers.27.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
260
+ "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
261
+ "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
262
+ "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
263
+ "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
264
+ "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
265
+ "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
266
+ "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
267
+ "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
268
+ "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
269
+ "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
270
+ "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
271
+ "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
272
+ "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
273
+ "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
274
+ "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
275
+ "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
276
+ "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
277
+ "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
278
+ "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
279
+ "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
280
+ "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
281
+ "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
282
+ "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
283
+ "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
284
+ "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
285
+ "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
286
+ "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
287
+ "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
288
+ "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
289
+ "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
290
+ "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
291
+ "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
292
+ "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
293
+ "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
294
+ "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
295
+ "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
296
+ "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
297
+ "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
298
+ "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
299
+ "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
300
+ "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
301
+ "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
302
+ "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
303
+ "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
304
+ "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
305
+ "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
306
+ "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
307
+ "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
308
+ "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
309
+ "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
310
+ "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
311
+ "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
312
+ "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
313
+ "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
314
+ "model.layers.7.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
315
+ "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
316
+ "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
317
+ "model.layers.7.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
318
+ "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
319
+ "model.layers.7.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
320
+ "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
321
+ "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
322
+ "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
323
+ "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
324
+ "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
325
+ "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
326
+ "model.layers.8.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
327
+ "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
328
+ "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
329
+ "model.layers.8.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
330
+ "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
331
+ "model.layers.8.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
332
+ "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
333
+ "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
334
+ "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
335
+ "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
336
+ "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
337
+ "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
338
+ "model.layers.9.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
339
+ "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
340
+ "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
341
+ "model.layers.9.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
342
+ "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
343
+ "model.layers.9.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
344
+ "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
345
+ "model.norm.weight": "model-00003-of-00004.safetensors"
346
+ }
347
+ }
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/special_tokens_map.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<|im_start|>",
4
+ "<|im_end|>",
5
+ "<|object_ref_start|>",
6
+ "<|object_ref_end|>",
7
+ "<|box_start|>",
8
+ "<|box_end|>",
9
+ "<|quad_start|>",
10
+ "<|quad_end|>",
11
+ "<|vision_start|>",
12
+ "<|vision_end|>",
13
+ "<|vision_pad|>",
14
+ "<|image_pad|>",
15
+ "<|video_pad|>"
16
+ ],
17
+ "eos_token": {
18
+ "content": "<|im_end|>",
19
+ "lstrip": false,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ },
24
+ "pad_token": {
25
+ "content": "<|endoftext|>",
26
+ "lstrip": false,
27
+ "normalized": false,
28
+ "rstrip": false,
29
+ "single_word": false
30
+ }
31
+ }
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
3
+ size 11421896
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/tokenizer_config.json ADDED
@@ -0,0 +1,207 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": false,
3
+ "add_prefix_space": false,
4
+ "added_tokens_decoder": {
5
+ "151643": {
6
+ "content": "<|endoftext|>",
7
+ "lstrip": false,
8
+ "normalized": false,
9
+ "rstrip": false,
10
+ "single_word": false,
11
+ "special": true
12
+ },
13
+ "151644": {
14
+ "content": "<|im_start|>",
15
+ "lstrip": false,
16
+ "normalized": false,
17
+ "rstrip": false,
18
+ "single_word": false,
19
+ "special": true
20
+ },
21
+ "151645": {
22
+ "content": "<|im_end|>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false,
27
+ "special": true
28
+ },
29
+ "151646": {
30
+ "content": "<|object_ref_start|>",
31
+ "lstrip": false,
32
+ "normalized": false,
33
+ "rstrip": false,
34
+ "single_word": false,
35
+ "special": true
36
+ },
37
+ "151647": {
38
+ "content": "<|object_ref_end|>",
39
+ "lstrip": false,
40
+ "normalized": false,
41
+ "rstrip": false,
42
+ "single_word": false,
43
+ "special": true
44
+ },
45
+ "151648": {
46
+ "content": "<|box_start|>",
47
+ "lstrip": false,
48
+ "normalized": false,
49
+ "rstrip": false,
50
+ "single_word": false,
51
+ "special": true
52
+ },
53
+ "151649": {
54
+ "content": "<|box_end|>",
55
+ "lstrip": false,
56
+ "normalized": false,
57
+ "rstrip": false,
58
+ "single_word": false,
59
+ "special": true
60
+ },
61
+ "151650": {
62
+ "content": "<|quad_start|>",
63
+ "lstrip": false,
64
+ "normalized": false,
65
+ "rstrip": false,
66
+ "single_word": false,
67
+ "special": true
68
+ },
69
+ "151651": {
70
+ "content": "<|quad_end|>",
71
+ "lstrip": false,
72
+ "normalized": false,
73
+ "rstrip": false,
74
+ "single_word": false,
75
+ "special": true
76
+ },
77
+ "151652": {
78
+ "content": "<|vision_start|>",
79
+ "lstrip": false,
80
+ "normalized": false,
81
+ "rstrip": false,
82
+ "single_word": false,
83
+ "special": true
84
+ },
85
+ "151653": {
86
+ "content": "<|vision_end|>",
87
+ "lstrip": false,
88
+ "normalized": false,
89
+ "rstrip": false,
90
+ "single_word": false,
91
+ "special": true
92
+ },
93
+ "151654": {
94
+ "content": "<|vision_pad|>",
95
+ "lstrip": false,
96
+ "normalized": false,
97
+ "rstrip": false,
98
+ "single_word": false,
99
+ "special": true
100
+ },
101
+ "151655": {
102
+ "content": "<|image_pad|>",
103
+ "lstrip": false,
104
+ "normalized": false,
105
+ "rstrip": false,
106
+ "single_word": false,
107
+ "special": true
108
+ },
109
+ "151656": {
110
+ "content": "<|video_pad|>",
111
+ "lstrip": false,
112
+ "normalized": false,
113
+ "rstrip": false,
114
+ "single_word": false,
115
+ "special": true
116
+ },
117
+ "151657": {
118
+ "content": "<tool_call>",
119
+ "lstrip": false,
120
+ "normalized": false,
121
+ "rstrip": false,
122
+ "single_word": false,
123
+ "special": false
124
+ },
125
+ "151658": {
126
+ "content": "</tool_call>",
127
+ "lstrip": false,
128
+ "normalized": false,
129
+ "rstrip": false,
130
+ "single_word": false,
131
+ "special": false
132
+ },
133
+ "151659": {
134
+ "content": "<|fim_prefix|>",
135
+ "lstrip": false,
136
+ "normalized": false,
137
+ "rstrip": false,
138
+ "single_word": false,
139
+ "special": false
140
+ },
141
+ "151660": {
142
+ "content": "<|fim_middle|>",
143
+ "lstrip": false,
144
+ "normalized": false,
145
+ "rstrip": false,
146
+ "single_word": false,
147
+ "special": false
148
+ },
149
+ "151661": {
150
+ "content": "<|fim_suffix|>",
151
+ "lstrip": false,
152
+ "normalized": false,
153
+ "rstrip": false,
154
+ "single_word": false,
155
+ "special": false
156
+ },
157
+ "151662": {
158
+ "content": "<|fim_pad|>",
159
+ "lstrip": false,
160
+ "normalized": false,
161
+ "rstrip": false,
162
+ "single_word": false,
163
+ "special": false
164
+ },
165
+ "151663": {
166
+ "content": "<|repo_name|>",
167
+ "lstrip": false,
168
+ "normalized": false,
169
+ "rstrip": false,
170
+ "single_word": false,
171
+ "special": false
172
+ },
173
+ "151664": {
174
+ "content": "<|file_sep|>",
175
+ "lstrip": false,
176
+ "normalized": false,
177
+ "rstrip": false,
178
+ "single_word": false,
179
+ "special": false
180
+ }
181
+ },
182
+ "additional_special_tokens": [
183
+ "<|im_start|>",
184
+ "<|im_end|>",
185
+ "<|object_ref_start|>",
186
+ "<|object_ref_end|>",
187
+ "<|box_start|>",
188
+ "<|box_end|>",
189
+ "<|quad_start|>",
190
+ "<|quad_end|>",
191
+ "<|vision_start|>",
192
+ "<|vision_end|>",
193
+ "<|vision_pad|>",
194
+ "<|image_pad|>",
195
+ "<|video_pad|>"
196
+ ],
197
+ "bos_token": null,
198
+ "clean_up_tokenization_spaces": false,
199
+ "eos_token": "<|im_end|>",
200
+ "errors": "replace",
201
+ "extra_special_tokens": {},
202
+ "model_max_length": 131072,
203
+ "pad_token": "<|endoftext|>",
204
+ "split_special_tokens": false,
205
+ "tokenizer_class": "Qwen2Tokenizer",
206
+ "unk_token": null
207
+ }
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/trainer_state.json ADDED
@@ -0,0 +1,731 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": 86,
3
+ "best_metric": 0.37377971,
4
+ "best_model_checkpoint": "/group/40143/hongzhuyi/ms-swift/output/v1-20250912-013615/checkpoint-86",
5
+ "epoch": 1.0,
6
+ "eval_steps": 2000.0,
7
+ "global_step": 86,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.011730205278592375,
14
+ "grad_norm": 19.42347886416143,
15
+ "learning_rate": 1.111111111111111e-07,
16
+ "loss": 0.9745279550552368,
17
+ "step": 1,
18
+ "token_acc": 0.7729915976524353
19
+ },
20
+ {
21
+ "epoch": 0.02346041055718475,
22
+ "grad_norm": 18.20102937211775,
23
+ "learning_rate": 2.222222222222222e-07,
24
+ "loss": 0.9701757431030273,
25
+ "step": 2,
26
+ "token_acc": 0.7739322185516357
27
+ },
28
+ {
29
+ "epoch": 0.03519061583577713,
30
+ "grad_norm": 19.78533819918654,
31
+ "learning_rate": 3.333333333333333e-07,
32
+ "loss": 1.0109223127365112,
33
+ "step": 3,
34
+ "token_acc": 0.7666413187980652
35
+ },
36
+ {
37
+ "epoch": 0.0469208211143695,
38
+ "grad_norm": 19.03486317488165,
39
+ "learning_rate": 4.444444444444444e-07,
40
+ "loss": 0.9887339472770691,
41
+ "step": 4,
42
+ "token_acc": 0.7682322859764099
43
+ },
44
+ {
45
+ "epoch": 0.05865102639296188,
46
+ "grad_norm": 17.64436521081935,
47
+ "learning_rate": 5.555555555555555e-07,
48
+ "loss": 0.94492107629776,
49
+ "step": 5,
50
+ "token_acc": 0.775278627872467
51
+ },
52
+ {
53
+ "epoch": 0.07038123167155426,
54
+ "grad_norm": 17.48642955713519,
55
+ "learning_rate": 6.666666666666666e-07,
56
+ "loss": 0.9331197738647461,
57
+ "step": 6,
58
+ "token_acc": 0.7808876037597656
59
+ },
60
+ {
61
+ "epoch": 0.08211143695014662,
62
+ "grad_norm": 17.401940797372863,
63
+ "learning_rate": 7.777777777777778e-07,
64
+ "loss": 0.9543712139129639,
65
+ "step": 7,
66
+ "token_acc": 0.7736433148384094
67
+ },
68
+ {
69
+ "epoch": 0.093841642228739,
70
+ "grad_norm": 14.675874021386344,
71
+ "learning_rate": 8.888888888888888e-07,
72
+ "loss": 0.9072513580322266,
73
+ "step": 8,
74
+ "token_acc": 0.7740758061408997
75
+ },
76
+ {
77
+ "epoch": 0.10557184750733138,
78
+ "grad_norm": 15.006000189143862,
79
+ "learning_rate": 1e-06,
80
+ "loss": 0.9155452251434326,
81
+ "step": 9,
82
+ "token_acc": 0.7733547687530518
83
+ },
84
+ {
85
+ "epoch": 0.11730205278592376,
86
+ "grad_norm": 9.152052949663721,
87
+ "learning_rate": 9.999071352056673e-07,
88
+ "loss": 0.8092969059944153,
89
+ "step": 10,
90
+ "token_acc": 0.7808704972267151
91
+ },
92
+ {
93
+ "epoch": 0.12903225806451613,
94
+ "grad_norm": 8.326679281131945,
95
+ "learning_rate": 9.996285753181497e-07,
96
+ "loss": 0.7987140417098999,
97
+ "step": 11,
98
+ "token_acc": 0.7814286947250366
99
+ },
100
+ {
101
+ "epoch": 0.14076246334310852,
102
+ "grad_norm": 7.439884752413428,
103
+ "learning_rate": 9.99164423811074e-07,
104
+ "loss": 0.7931578159332275,
105
+ "step": 12,
106
+ "token_acc": 0.7772960066795349
107
+ },
108
+ {
109
+ "epoch": 0.15249266862170088,
110
+ "grad_norm": 7.119023890418445,
111
+ "learning_rate": 9.985148530977764e-07,
112
+ "loss": 0.7600352764129639,
113
+ "step": 13,
114
+ "token_acc": 0.7844690680503845
115
+ },
116
+ {
117
+ "epoch": 0.16422287390029325,
118
+ "grad_norm": 5.035964456988531,
119
+ "learning_rate": 9.976801044672607e-07,
120
+ "loss": 0.6961469650268555,
121
+ "step": 14,
122
+ "token_acc": 0.7958953976631165
123
+ },
124
+ {
125
+ "epoch": 0.17595307917888564,
126
+ "grad_norm": 5.1530837703132075,
127
+ "learning_rate": 9.966604879945656e-07,
128
+ "loss": 0.6827518343925476,
129
+ "step": 15,
130
+ "token_acc": 0.7969183921813965
131
+ },
132
+ {
133
+ "epoch": 0.187683284457478,
134
+ "grad_norm": 4.578053887212758,
135
+ "learning_rate": 9.954563824255877e-07,
136
+ "loss": 0.6470961570739746,
137
+ "step": 16,
138
+ "token_acc": 0.8064683079719543
139
+ },
140
+ {
141
+ "epoch": 0.19941348973607037,
142
+ "grad_norm": 4.471874336838139,
143
+ "learning_rate": 9.94068235036391e-07,
144
+ "loss": 0.6799627542495728,
145
+ "step": 17,
146
+ "token_acc": 0.8002287149429321
147
+ },
148
+ {
149
+ "epoch": 0.21114369501466276,
150
+ "grad_norm": 4.5065795595929705,
151
+ "learning_rate": 9.924965614670628e-07,
152
+ "loss": 0.6805820465087891,
153
+ "step": 18,
154
+ "token_acc": 0.8004816770553589
155
+ },
156
+ {
157
+ "epoch": 0.22287390029325513,
158
+ "grad_norm": 4.026774039902112,
159
+ "learning_rate": 9.90741945530174e-07,
160
+ "loss": 0.6293234825134277,
161
+ "step": 19,
162
+ "token_acc": 0.8133951425552368
163
+ },
164
+ {
165
+ "epoch": 0.23460410557184752,
166
+ "grad_norm": 4.1944187169445035,
167
+ "learning_rate": 9.888050389939172e-07,
168
+ "loss": 0.6192097067832947,
169
+ "step": 20,
170
+ "token_acc": 0.8118802905082703
171
+ },
172
+ {
173
+ "epoch": 0.24633431085043989,
174
+ "grad_norm": 3.826379198058098,
175
+ "learning_rate": 9.866865613400006e-07,
176
+ "loss": 0.6224480867385864,
177
+ "step": 21,
178
+ "token_acc": 0.8149859309196472
179
+ },
180
+ {
181
+ "epoch": 0.25806451612903225,
182
+ "grad_norm": 3.7571032828466873,
183
+ "learning_rate": 9.843872994963912e-07,
184
+ "loss": 0.655880331993103,
185
+ "step": 22,
186
+ "token_acc": 0.8066907525062561
187
+ },
188
+ {
189
+ "epoch": 0.2697947214076246,
190
+ "grad_norm": 3.681409260856744,
191
+ "learning_rate": 9.819081075450013e-07,
192
+ "loss": 0.5886544585227966,
193
+ "step": 23,
194
+ "token_acc": 0.8210309743881226
195
+ },
196
+ {
197
+ "epoch": 0.28152492668621704,
198
+ "grad_norm": 3.8664356748850275,
199
+ "learning_rate": 9.792499064044342e-07,
200
+ "loss": 0.5898054838180542,
201
+ "step": 24,
202
+ "token_acc": 0.819602906703949
203
+ },
204
+ {
205
+ "epoch": 0.2932551319648094,
206
+ "grad_norm": 3.769342936970688,
207
+ "learning_rate": 9.764136834878985e-07,
208
+ "loss": 0.5778634548187256,
209
+ "step": 25,
210
+ "token_acc": 0.8214534521102905
211
+ },
212
+ {
213
+ "epoch": 0.30498533724340177,
214
+ "grad_norm": 3.311750534405761,
215
+ "learning_rate": 9.734004923364256e-07,
216
+ "loss": 0.5714818239212036,
217
+ "step": 26,
218
+ "token_acc": 0.8214825987815857
219
+ },
220
+ {
221
+ "epoch": 0.31671554252199413,
222
+ "grad_norm": 3.2183065918077927,
223
+ "learning_rate": 9.702114522275216e-07,
224
+ "loss": 0.5601825714111328,
225
+ "step": 27,
226
+ "token_acc": 0.8292351365089417
227
+ },
228
+ {
229
+ "epoch": 0.3284457478005865,
230
+ "grad_norm": 2.7897603795961583,
231
+ "learning_rate": 9.66847747759402e-07,
232
+ "loss": 0.527702808380127,
233
+ "step": 28,
234
+ "token_acc": 0.835481584072113
235
+ },
236
+ {
237
+ "epoch": 0.34017595307917886,
238
+ "grad_norm": 2.847650852465056,
239
+ "learning_rate": 9.63310628410961e-07,
240
+ "loss": 0.5497344136238098,
241
+ "step": 29,
242
+ "token_acc": 0.8285976648330688
243
+ },
244
+ {
245
+ "epoch": 0.3519061583577713,
246
+ "grad_norm": 2.705846258459626,
247
+ "learning_rate": 9.596014080776421e-07,
248
+ "loss": 0.50850510597229,
249
+ "step": 30,
250
+ "token_acc": 0.8413625359535217
251
+ },
252
+ {
253
+ "epoch": 0.36363636363636365,
254
+ "grad_norm": 2.4259522658265134,
255
+ "learning_rate": 9.55721464583379e-07,
256
+ "loss": 0.5382215976715088,
257
+ "step": 31,
258
+ "token_acc": 0.8332553505897522
259
+ },
260
+ {
261
+ "epoch": 0.375366568914956,
262
+ "grad_norm": 2.571450563187992,
263
+ "learning_rate": 9.516722391687902e-07,
264
+ "loss": 0.5567100644111633,
265
+ "step": 32,
266
+ "token_acc": 0.825095534324646
267
+ },
268
+ {
269
+ "epoch": 0.3870967741935484,
270
+ "grad_norm": 2.4023357046065903,
271
+ "learning_rate": 9.474552359558165e-07,
272
+ "loss": 0.5305403470993042,
273
+ "step": 33,
274
+ "token_acc": 0.8326757550239563
275
+ },
276
+ {
277
+ "epoch": 0.39882697947214074,
278
+ "grad_norm": 2.3633434913754563,
279
+ "learning_rate": 9.430720213890029e-07,
280
+ "loss": 0.5253998041152954,
281
+ "step": 34,
282
+ "token_acc": 0.8343205451965332
283
+ },
284
+ {
285
+ "epoch": 0.41055718475073316,
286
+ "grad_norm": 2.4155773004795615,
287
+ "learning_rate": 9.385242236536259e-07,
288
+ "loss": 0.5054779052734375,
289
+ "step": 35,
290
+ "token_acc": 0.8360969424247742
291
+ },
292
+ {
293
+ "epoch": 0.4222873900293255,
294
+ "grad_norm": 2.1668550281375953,
295
+ "learning_rate": 9.338135320708911e-07,
296
+ "loss": 0.5100793838500977,
297
+ "step": 36,
298
+ "token_acc": 0.8403894305229187
299
+ },
300
+ {
301
+ "epoch": 0.4340175953079179,
302
+ "grad_norm": 2.2142817682644336,
303
+ "learning_rate": 9.289416964704185e-07,
304
+ "loss": 0.5320375561714172,
305
+ "step": 37,
306
+ "token_acc": 0.8320334553718567
307
+ },
308
+ {
309
+ "epoch": 0.44574780058651026,
310
+ "grad_norm": 2.231366943821694,
311
+ "learning_rate": 9.239105265402525e-07,
312
+ "loss": 0.5052040815353394,
313
+ "step": 38,
314
+ "token_acc": 0.8394160866737366
315
+ },
316
+ {
317
+ "epoch": 0.4574780058651026,
318
+ "grad_norm": 2.1947282253307785,
319
+ "learning_rate": 9.187218911546361e-07,
320
+ "loss": 0.5045727491378784,
321
+ "step": 39,
322
+ "token_acc": 0.8403294086456299
323
+ },
324
+ {
325
+ "epoch": 0.46920821114369504,
326
+ "grad_norm": 2.082686891153019,
327
+ "learning_rate": 9.133777176798012e-07,
328
+ "loss": 0.4821414649486542,
329
+ "step": 40,
330
+ "token_acc": 0.8482713103294373
331
+ },
332
+ {
333
+ "epoch": 0.4809384164222874,
334
+ "grad_norm": 2.1147451850371293,
335
+ "learning_rate": 9.078799912580303e-07,
336
+ "loss": 0.49139469861984253,
337
+ "step": 41,
338
+ "token_acc": 0.8435467481613159
339
+ },
340
+ {
341
+ "epoch": 0.49266862170087977,
342
+ "grad_norm": 2.0542615188891578,
343
+ "learning_rate": 9.022307540702576e-07,
344
+ "loss": 0.495099276304245,
345
+ "step": 42,
346
+ "token_acc": 0.8379960656166077
347
+ },
348
+ {
349
+ "epoch": 0.5043988269794721,
350
+ "grad_norm": 2.0136030676549392,
351
+ "learning_rate": 8.964321045774806e-07,
352
+ "loss": 0.489553302526474,
353
+ "step": 43,
354
+ "token_acc": 0.8405314087867737
355
+ },
356
+ {
357
+ "epoch": 0.5161290322580645,
358
+ "grad_norm": 1.9630127736815466,
359
+ "learning_rate": 8.904861967412701e-07,
360
+ "loss": 0.4687340557575226,
361
+ "step": 44,
362
+ "token_acc": 0.8476594090461731
363
+ },
364
+ {
365
+ "epoch": 0.5278592375366569,
366
+ "grad_norm": 1.9513540796416378,
367
+ "learning_rate": 8.843952392236593e-07,
368
+ "loss": 0.4688946306705475,
369
+ "step": 45,
370
+ "token_acc": 0.8495379090309143
371
+ },
372
+ {
373
+ "epoch": 0.5395894428152492,
374
+ "grad_norm": 2.0000887958335443,
375
+ "learning_rate": 8.781614945667168e-07,
376
+ "loss": 0.459961861371994,
377
+ "step": 46,
378
+ "token_acc": 0.8521719574928284
379
+ },
380
+ {
381
+ "epoch": 0.5513196480938416,
382
+ "grad_norm": 1.9642568842781296,
383
+ "learning_rate": 8.717872783521047e-07,
384
+ "loss": 0.48166409134864807,
385
+ "step": 47,
386
+ "token_acc": 0.8472545146942139
387
+ },
388
+ {
389
+ "epoch": 0.5630498533724341,
390
+ "grad_norm": 1.9787796147862529,
391
+ "learning_rate": 8.652749583409339e-07,
392
+ "loss": 0.47355878353118896,
393
+ "step": 48,
394
+ "token_acc": 0.8466734886169434
395
+ },
396
+ {
397
+ "epoch": 0.5747800586510264,
398
+ "grad_norm": 1.9308160364727027,
399
+ "learning_rate": 8.586269535942385e-07,
400
+ "loss": 0.4618028998374939,
401
+ "step": 49,
402
+ "token_acc": 0.8513069152832031
403
+ },
404
+ {
405
+ "epoch": 0.5865102639296188,
406
+ "grad_norm": 1.8562690832047652,
407
+ "learning_rate": 8.518457335743924e-07,
408
+ "loss": 0.4850979447364807,
409
+ "step": 50,
410
+ "token_acc": 0.8428641557693481
411
+ },
412
+ {
413
+ "epoch": 0.5982404692082112,
414
+ "grad_norm": 1.8213744008266877,
415
+ "learning_rate": 8.449338172278058e-07,
416
+ "loss": 0.4308124780654907,
417
+ "step": 51,
418
+ "token_acc": 0.8571488857269287
419
+ },
420
+ {
421
+ "epoch": 0.6099706744868035,
422
+ "grad_norm": 1.9045019401040395,
423
+ "learning_rate": 8.378937720492383e-07,
424
+ "loss": 0.4853397607803345,
425
+ "step": 52,
426
+ "token_acc": 0.8429522514343262
427
+ },
428
+ {
429
+ "epoch": 0.6217008797653959,
430
+ "grad_norm": 1.8218123277369342,
431
+ "learning_rate": 8.307282131280804e-07,
432
+ "loss": 0.4712865352630615,
433
+ "step": 53,
434
+ "token_acc": 0.8480793833732605
435
+ },
436
+ {
437
+ "epoch": 0.6334310850439883,
438
+ "grad_norm": 1.8207444397924821,
439
+ "learning_rate": 8.23439802176954e-07,
440
+ "loss": 0.4410493075847626,
441
+ "step": 54,
442
+ "token_acc": 0.85650634765625
443
+ },
444
+ {
445
+ "epoch": 0.6451612903225806,
446
+ "grad_norm": 1.7646262400395012,
447
+ "learning_rate": 8.160312465429952e-07,
448
+ "loss": 0.45616358518600464,
449
+ "step": 55,
450
+ "token_acc": 0.8514736890792847
451
+ },
452
+ {
453
+ "epoch": 0.656891495601173,
454
+ "grad_norm": 1.859319205117003,
455
+ "learning_rate": 8.085052982021847e-07,
456
+ "loss": 0.4499448537826538,
457
+ "step": 56,
458
+ "token_acc": 0.8548539876937866
459
+ },
460
+ {
461
+ "epoch": 0.6686217008797654,
462
+ "grad_norm": 1.6613176216022334,
463
+ "learning_rate": 8.008647527371022e-07,
464
+ "loss": 0.43093234300613403,
465
+ "step": 57,
466
+ "token_acc": 0.8587403893470764
467
+ },
468
+ {
469
+ "epoch": 0.6803519061583577,
470
+ "grad_norm": 1.7104406984196343,
471
+ "learning_rate": 7.931124482984801e-07,
472
+ "loss": 0.46069544553756714,
473
+ "step": 58,
474
+ "token_acc": 0.8481629490852356
475
+ },
476
+ {
477
+ "epoch": 0.6920821114369502,
478
+ "grad_norm": 1.8009260213577374,
479
+ "learning_rate": 7.85251264550948e-07,
480
+ "loss": 0.45625898241996765,
481
+ "step": 59,
482
+ "token_acc": 0.8500686287879944
483
+ },
484
+ {
485
+ "epoch": 0.7038123167155426,
486
+ "grad_norm": 1.699888333456424,
487
+ "learning_rate": 7.772841216033532e-07,
488
+ "loss": 0.46078723669052124,
489
+ "step": 60,
490
+ "token_acc": 0.8482969403266907
491
+ },
492
+ {
493
+ "epoch": 0.7155425219941349,
494
+ "grad_norm": 1.738809601837965,
495
+ "learning_rate": 7.69213978924061e-07,
496
+ "loss": 0.4371066987514496,
497
+ "step": 61,
498
+ "token_acc": 0.8570924401283264
499
+ },
500
+ {
501
+ "epoch": 0.7272727272727273,
502
+ "grad_norm": 1.6763935168681205,
503
+ "learning_rate": 7.610438342416319e-07,
504
+ "loss": 0.4535030722618103,
505
+ "step": 62,
506
+ "token_acc": 0.8520426750183105
507
+ },
508
+ {
509
+ "epoch": 0.7390029325513197,
510
+ "grad_norm": 1.6971447873397694,
511
+ "learning_rate": 7.527767224312882e-07,
512
+ "loss": 0.4411541521549225,
513
+ "step": 63,
514
+ "token_acc": 0.854651153087616
515
+ },
516
+ {
517
+ "epoch": 0.750733137829912,
518
+ "grad_norm": 1.7785564815624373,
519
+ "learning_rate": 7.444157143875819e-07,
520
+ "loss": 0.43432003259658813,
521
+ "step": 64,
522
+ "token_acc": 0.8572342395782471
523
+ },
524
+ {
525
+ "epoch": 0.7624633431085044,
526
+ "grad_norm": 1.7516774946326659,
527
+ "learning_rate": 7.359639158836827e-07,
528
+ "loss": 0.4576050937175751,
529
+ "step": 65,
530
+ "token_acc": 0.852763831615448
531
+ },
532
+ {
533
+ "epoch": 0.7741935483870968,
534
+ "grad_norm": 1.8159173350443534,
535
+ "learning_rate": 7.274244664177097e-07,
536
+ "loss": 0.4335535764694214,
537
+ "step": 66,
538
+ "token_acc": 0.8581987023353577
539
+ },
540
+ {
541
+ "epoch": 0.7859237536656891,
542
+ "grad_norm": 1.6985715890277628,
543
+ "learning_rate": 7.188005380465364e-07,
544
+ "loss": 0.43045544624328613,
545
+ "step": 67,
546
+ "token_acc": 0.8622540831565857
547
+ },
548
+ {
549
+ "epoch": 0.7976539589442815,
550
+ "grad_norm": 1.6425422999936439,
551
+ "learning_rate": 7.100953342075009e-07,
552
+ "loss": 0.4037867486476898,
553
+ "step": 68,
554
+ "token_acc": 0.8682027459144592
555
+ },
556
+ {
557
+ "epoch": 0.8093841642228738,
558
+ "grad_norm": 1.729981054732428,
559
+ "learning_rate": 7.013120885284598e-07,
560
+ "loss": 0.45598796010017395,
561
+ "step": 69,
562
+ "token_acc": 0.8521715998649597
563
+ },
564
+ {
565
+ "epoch": 0.8211143695014663,
566
+ "grad_norm": 1.7637112650059394,
567
+ "learning_rate": 6.924540636266272e-07,
568
+ "loss": 0.44459569454193115,
569
+ "step": 70,
570
+ "token_acc": 0.854467511177063
571
+ },
572
+ {
573
+ "epoch": 0.8328445747800587,
574
+ "grad_norm": 1.7887656982582423,
575
+ "learning_rate": 6.83524549896646e-07,
576
+ "loss": 0.45404496788978577,
577
+ "step": 71,
578
+ "token_acc": 0.851087749004364
579
+ },
580
+ {
581
+ "epoch": 0.844574780058651,
582
+ "grad_norm": 1.6852945606800631,
583
+ "learning_rate": 6.745268642883404e-07,
584
+ "loss": 0.4606991708278656,
585
+ "step": 72,
586
+ "token_acc": 0.8486180305480957
587
+ },
588
+ {
589
+ "epoch": 0.8563049853372434,
590
+ "grad_norm": 1.6399899629621875,
591
+ "learning_rate": 6.654643490746041e-07,
592
+ "loss": 0.45651188492774963,
593
+ "step": 73,
594
+ "token_acc": 0.8511216640472412
595
+ },
596
+ {
597
+ "epoch": 0.8680351906158358,
598
+ "grad_norm": 1.738549808935775,
599
+ "learning_rate": 6.563403706098832e-07,
600
+ "loss": 0.44438982009887695,
601
+ "step": 74,
602
+ "token_acc": 0.8562652468681335
603
+ },
604
+ {
605
+ "epoch": 0.8797653958944281,
606
+ "grad_norm": 1.7269017599894254,
607
+ "learning_rate": 6.47158318079712e-07,
608
+ "loss": 0.41442689299583435,
609
+ "step": 75,
610
+ "token_acc": 0.8645210862159729
611
+ },
612
+ {
613
+ "epoch": 0.8914956011730205,
614
+ "grad_norm": 1.7743815830820935,
615
+ "learning_rate": 6.379216022417695e-07,
616
+ "loss": 0.42937588691711426,
617
+ "step": 76,
618
+ "token_acc": 0.8610733151435852
619
+ },
620
+ {
621
+ "epoch": 0.9032258064516129,
622
+ "grad_norm": 1.5857889981196076,
623
+ "learning_rate": 6.286336541589223e-07,
624
+ "loss": 0.4185515344142914,
625
+ "step": 77,
626
+ "token_acc": 0.863837718963623
627
+ },
628
+ {
629
+ "epoch": 0.9149560117302052,
630
+ "grad_norm": 1.6775726033449394,
631
+ "learning_rate": 6.192979239247242e-07,
632
+ "loss": 0.39629387855529785,
633
+ "step": 78,
634
+ "token_acc": 0.8690212368965149
635
+ },
636
+ {
637
+ "epoch": 0.9266862170087976,
638
+ "grad_norm": 1.6671136884891102,
639
+ "learning_rate": 6.099178793818478e-07,
640
+ "loss": 0.4317322075366974,
641
+ "step": 79,
642
+ "token_acc": 0.8624503016471863
643
+ },
644
+ {
645
+ "epoch": 0.9384164222873901,
646
+ "grad_norm": 1.6320376815696551,
647
+ "learning_rate": 6.004970048339225e-07,
648
+ "loss": 0.4439336657524109,
649
+ "step": 80,
650
+ "token_acc": 0.8561707139015198
651
+ },
652
+ {
653
+ "epoch": 0.9501466275659824,
654
+ "grad_norm": 1.6010567395710715,
655
+ "learning_rate": 5.910387997512573e-07,
656
+ "loss": 0.43129977583885193,
657
+ "step": 81,
658
+ "token_acc": 0.8600221872329712
659
+ },
660
+ {
661
+ "epoch": 0.9618768328445748,
662
+ "grad_norm": 1.6137708142260028,
663
+ "learning_rate": 5.815467774709313e-07,
664
+ "loss": 0.40753841400146484,
665
+ "step": 82,
666
+ "token_acc": 0.8669683337211609
667
+ },
668
+ {
669
+ "epoch": 0.9736070381231672,
670
+ "grad_norm": 1.71404162588074,
671
+ "learning_rate": 5.720244638917323e-07,
672
+ "loss": 0.40983232855796814,
673
+ "step": 83,
674
+ "token_acc": 0.8663516044616699
675
+ },
676
+ {
677
+ "epoch": 0.9853372434017595,
678
+ "grad_norm": 1.7287418122113773,
679
+ "learning_rate": 5.624753961644281e-07,
680
+ "loss": 0.43514540791511536,
681
+ "step": 84,
682
+ "token_acc": 0.8586166501045227
683
+ },
684
+ {
685
+ "epoch": 0.9970674486803519,
686
+ "grad_norm": 1.7083837850323478,
687
+ "learning_rate": 5.529031213778614e-07,
688
+ "loss": 0.4181726574897766,
689
+ "step": 85,
690
+ "token_acc": 0.8610889911651611
691
+ },
692
+ {
693
+ "epoch": 1.0,
694
+ "grad_norm": 4.682383086910253,
695
+ "learning_rate": 5.433111952413494e-07,
696
+ "loss": 0.437732994556427,
697
+ "step": 86,
698
+ "token_acc": 0.8590138554573059
699
+ },
700
+ {
701
+ "epoch": 1.0,
702
+ "eval_loss": 0.3737797141075134,
703
+ "eval_runtime": 4.3026,
704
+ "eval_samples_per_second": 1.162,
705
+ "eval_steps_per_second": 0.232,
706
+ "eval_token_acc": 0.8611544370651245,
707
+ "step": 86
708
+ }
709
+ ],
710
+ "logging_steps": 1,
711
+ "max_steps": 172,
712
+ "num_input_tokens_seen": 0,
713
+ "num_train_epochs": 2,
714
+ "save_steps": 500,
715
+ "stateful_callbacks": {
716
+ "TrainerControl": {
717
+ "args": {
718
+ "should_epoch_stop": false,
719
+ "should_evaluate": false,
720
+ "should_log": false,
721
+ "should_save": true,
722
+ "should_training_stop": false
723
+ },
724
+ "attributes": {}
725
+ }
726
+ },
727
+ "total_flos": 100849815961600.0,
728
+ "train_batch_size": 2,
729
+ "trial_name": null,
730
+ "trial_params": null
731
+ }
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:59069f1b6b198eaea104863a28f5a6f6ab3f67712636ca0246f3165e68c62374
3
+ size 9041
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/vocab.json ADDED
The diff for this file is too large to render. See raw diff
 
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/checkpoint-86/zero_to_fp32.py ADDED
@@ -0,0 +1,760 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python
2
+
3
+ # Copyright (c) Microsoft Corporation.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ # DeepSpeed Team
7
+
8
+ # This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
9
+ # copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
10
+ # the future. Once extracted, the weights don't require DeepSpeed and can be used in any
11
+ # application.
12
+ #
13
+ # example:
14
+ # python zero_to_fp32.py . output_dir/
15
+ # or
16
+ # python zero_to_fp32.py . output_dir/ --safe_serialization
17
+
18
+ import argparse
19
+ import torch
20
+ import glob
21
+ import math
22
+ import os
23
+ import re
24
+ import gc
25
+ import json
26
+ import numpy as np
27
+ from tqdm import tqdm
28
+ from collections import OrderedDict
29
+ from dataclasses import dataclass
30
+
31
+ # while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
32
+ # DeepSpeed data structures it has to be available in the current python environment.
33
+ from deepspeed.utils import logger
34
+ from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
35
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
36
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
37
+
38
+
39
+ @dataclass
40
+ class zero_model_state:
41
+ buffers: dict()
42
+ param_shapes: dict()
43
+ shared_params: list
44
+ ds_version: int
45
+ frozen_param_shapes: dict()
46
+ frozen_param_fragments: dict()
47
+
48
+
49
+ debug = 0
50
+
51
+ # load to cpu
52
+ device = torch.device('cpu')
53
+
54
+
55
+ def atoi(text):
56
+ return int(text) if text.isdigit() else text
57
+
58
+
59
+ def natural_keys(text):
60
+ '''
61
+ alist.sort(key=natural_keys) sorts in human order
62
+ http://nedbatchelder.com/blog/200712/human_sorting.html
63
+ (See Toothy's implementation in the comments)
64
+ '''
65
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
66
+
67
+
68
+ def get_model_state_file(checkpoint_dir, zero_stage):
69
+ if not os.path.isdir(checkpoint_dir):
70
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
71
+
72
+ # there should be only one file
73
+ if zero_stage <= 2:
74
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
75
+ elif zero_stage == 3:
76
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
77
+
78
+ if not os.path.exists(file):
79
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
80
+
81
+ return file
82
+
83
+
84
+ def get_checkpoint_files(checkpoint_dir, glob_pattern):
85
+ # XXX: need to test that this simple glob rule works for multi-node setup too
86
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
87
+
88
+ if len(ckpt_files) == 0:
89
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
90
+
91
+ return ckpt_files
92
+
93
+
94
+ def get_optim_files(checkpoint_dir):
95
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
96
+
97
+
98
+ def get_model_state_files(checkpoint_dir):
99
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
100
+
101
+
102
+ def parse_model_states(files):
103
+ zero_model_states = []
104
+ for file in files:
105
+ state_dict = torch.load(file, map_location=device, weights_only=False)
106
+
107
+ if BUFFER_NAMES not in state_dict:
108
+ raise ValueError(f"{file} is not a model state checkpoint")
109
+ buffer_names = state_dict[BUFFER_NAMES]
110
+ if debug:
111
+ print("Found buffers:", buffer_names)
112
+
113
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
114
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
115
+ param_shapes = state_dict[PARAM_SHAPES]
116
+
117
+ # collect parameters that are included in param_shapes
118
+ param_names = []
119
+ for s in param_shapes:
120
+ for name in s.keys():
121
+ param_names.append(name)
122
+
123
+ # update with frozen parameters
124
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
125
+ if frozen_param_shapes is not None:
126
+ if debug:
127
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
128
+ param_names += list(frozen_param_shapes.keys())
129
+
130
+ # handle shared params
131
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
132
+
133
+ ds_version = state_dict.get(DS_VERSION, None)
134
+
135
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
136
+
137
+ z_model_state = zero_model_state(buffers=buffers,
138
+ param_shapes=param_shapes,
139
+ shared_params=shared_params,
140
+ ds_version=ds_version,
141
+ frozen_param_shapes=frozen_param_shapes,
142
+ frozen_param_fragments=frozen_param_fragments)
143
+ zero_model_states.append(z_model_state)
144
+
145
+ return zero_model_states
146
+
147
+
148
+ def parse_optim_states(files, ds_checkpoint_dir):
149
+ total_files = len(files)
150
+ state_dicts = []
151
+ for f in tqdm(files, desc='Loading checkpoint shards'):
152
+ state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
153
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
154
+ # and also handle the case where it was already removed by another helper script
155
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
156
+ state_dicts.append(state_dict)
157
+
158
+ if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
159
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
160
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
161
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
162
+
163
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
164
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
165
+ # use the max of the partition_count to get the dp world_size.
166
+
167
+ if type(world_size) is list:
168
+ world_size = max(world_size)
169
+
170
+ if world_size != total_files:
171
+ raise ValueError(
172
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
173
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
174
+ )
175
+
176
+ # the groups are named differently in each stage
177
+ if zero_stage <= 2:
178
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
179
+ elif zero_stage == 3:
180
+ fp32_groups_key = FP32_FLAT_GROUPS
181
+ else:
182
+ raise ValueError(f"unknown zero stage {zero_stage}")
183
+
184
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
185
+ return zero_stage, world_size, fp32_flat_groups
186
+
187
+
188
+ def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
189
+ """
190
+ Returns fp32 state_dict reconstructed from ds checkpoint
191
+
192
+ Args:
193
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
194
+
195
+ """
196
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
197
+
198
+ optim_files = get_optim_files(ds_checkpoint_dir)
199
+ zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
200
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
201
+
202
+ model_files = get_model_state_files(ds_checkpoint_dir)
203
+
204
+ zero_model_states = parse_model_states(model_files)
205
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
206
+
207
+ if zero_stage <= 2:
208
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
209
+ exclude_frozen_parameters)
210
+ elif zero_stage == 3:
211
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
212
+ exclude_frozen_parameters)
213
+
214
+
215
+ def _zero2_merge_frozen_params(state_dict, zero_model_states):
216
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
217
+ return
218
+
219
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
220
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
221
+
222
+ if debug:
223
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
224
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
225
+
226
+ wanted_params = len(frozen_param_shapes)
227
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
228
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
229
+ print(f'Frozen params: Have {avail_numel} numels to process.')
230
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
231
+
232
+ total_params = 0
233
+ total_numel = 0
234
+ for name, shape in frozen_param_shapes.items():
235
+ total_params += 1
236
+ unpartitioned_numel = shape.numel()
237
+ total_numel += unpartitioned_numel
238
+
239
+ state_dict[name] = frozen_param_fragments[name]
240
+
241
+ if debug:
242
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
243
+
244
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
245
+
246
+
247
+ def _has_callable(obj, fn):
248
+ attr = getattr(obj, fn, None)
249
+ return callable(attr)
250
+
251
+
252
+ def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
253
+ param_shapes = zero_model_states[0].param_shapes
254
+
255
+ # Reconstruction protocol:
256
+ #
257
+ # XXX: document this
258
+
259
+ if debug:
260
+ for i in range(world_size):
261
+ for j in range(len(fp32_flat_groups[0])):
262
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
263
+
264
+ # XXX: memory usage doubles here (zero2)
265
+ num_param_groups = len(fp32_flat_groups[0])
266
+ merged_single_partition_of_fp32_groups = []
267
+ for i in range(num_param_groups):
268
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
269
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
270
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
271
+ avail_numel = sum(
272
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
273
+
274
+ if debug:
275
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
276
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
277
+ # not asserting if there is a mismatch due to possible padding
278
+ print(f"Have {avail_numel} numels to process.")
279
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
280
+
281
+ # params
282
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
283
+ # out-of-core computing solution
284
+ total_numel = 0
285
+ total_params = 0
286
+ for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
287
+ offset = 0
288
+ avail_numel = full_single_fp32_vector.numel()
289
+ for name, shape in shapes.items():
290
+
291
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
292
+ total_numel += unpartitioned_numel
293
+ total_params += 1
294
+
295
+ if debug:
296
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
297
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
298
+ offset += unpartitioned_numel
299
+
300
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
301
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
302
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
303
+ # live optimizer object, so we are checking that the numbers are within the right range
304
+ align_to = 2 * world_size
305
+
306
+ def zero2_align(x):
307
+ return align_to * math.ceil(x / align_to)
308
+
309
+ if debug:
310
+ print(f"original offset={offset}, avail_numel={avail_numel}")
311
+
312
+ offset = zero2_align(offset)
313
+ avail_numel = zero2_align(avail_numel)
314
+
315
+ if debug:
316
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
317
+
318
+ # Sanity check
319
+ if offset != avail_numel:
320
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
321
+
322
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
323
+
324
+
325
+ def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
326
+ exclude_frozen_parameters):
327
+ state_dict = OrderedDict()
328
+
329
+ # buffers
330
+ buffers = zero_model_states[0].buffers
331
+ state_dict.update(buffers)
332
+ if debug:
333
+ print(f"added {len(buffers)} buffers")
334
+
335
+ if not exclude_frozen_parameters:
336
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
337
+
338
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
339
+
340
+ # recover shared parameters
341
+ for pair in zero_model_states[0].shared_params:
342
+ if pair[1] in state_dict:
343
+ state_dict[pair[0]] = state_dict[pair[1]]
344
+
345
+ return state_dict
346
+
347
+
348
+ def zero3_partitioned_param_info(unpartitioned_numel, world_size):
349
+ remainder = unpartitioned_numel % world_size
350
+ padding_numel = (world_size - remainder) if remainder else 0
351
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
352
+ return partitioned_numel, padding_numel
353
+
354
+
355
+ def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
356
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
357
+ return
358
+
359
+ if debug:
360
+ for i in range(world_size):
361
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
362
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
363
+
364
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
365
+ wanted_params = len(frozen_param_shapes)
366
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
367
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
368
+ print(f'Frozen params: Have {avail_numel} numels to process.')
369
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
370
+
371
+ total_params = 0
372
+ total_numel = 0
373
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
374
+ total_params += 1
375
+ unpartitioned_numel = shape.numel()
376
+ total_numel += unpartitioned_numel
377
+
378
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
379
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
380
+
381
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
382
+
383
+ if debug:
384
+ print(
385
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
386
+ )
387
+
388
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
389
+
390
+
391
+ class GatheredTensor:
392
+ """
393
+ A pseudo tensor that collects partitioned weights.
394
+ It is more memory efficient when there are multiple groups.
395
+ """
396
+
397
+ def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
398
+ self.flat_groups = flat_groups
399
+ self.flat_groups_offset = flat_groups_offset
400
+ self.offset = offset
401
+ self.partitioned_numel = partitioned_numel
402
+ self.shape = shape
403
+ self.dtype = self.flat_groups[0][0].dtype
404
+
405
+ def contiguous(self):
406
+ """
407
+ Merge partitioned weights from flat_groups into a single tensor.
408
+ """
409
+ end_idx = self.offset + self.partitioned_numel
410
+ world_size = len(self.flat_groups)
411
+ pad_flat_param_chunks = []
412
+
413
+ for rank_i in range(world_size):
414
+ # for each rank, we need to collect weights from related group/groups
415
+ flat_groups_at_rank_i = self.flat_groups[rank_i]
416
+ start_group_id = None
417
+ end_group_id = None
418
+ for group_id in range(len(self.flat_groups_offset)):
419
+ if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
420
+ start_group_id = group_id
421
+ if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
422
+ end_group_id = group_id
423
+ break
424
+ # collect weights from related group/groups
425
+ for group_id in range(start_group_id, end_group_id + 1):
426
+ flat_tensor = flat_groups_at_rank_i[group_id]
427
+ start_offset = self.offset - self.flat_groups_offset[group_id]
428
+ end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
429
+ pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
430
+
431
+ # collect weights from all ranks
432
+ pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
433
+ param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
434
+ return param
435
+
436
+
437
+ def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
438
+ param_shapes = zero_model_states[0].param_shapes
439
+ avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
440
+
441
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
442
+ # param, re-consolidating each param, while dealing with padding if any
443
+
444
+ # merge list of dicts, preserving order
445
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
446
+
447
+ if debug:
448
+ for i in range(world_size):
449
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
450
+
451
+ wanted_params = len(param_shapes)
452
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
453
+ # not asserting if there is a mismatch due to possible padding
454
+ avail_numel = fp32_flat_groups[0].numel() * world_size
455
+ print(f"Trainable params: Have {avail_numel} numels to process.")
456
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
457
+
458
+ # params
459
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
460
+ # out-of-core computing solution
461
+ offset = 0
462
+ total_numel = 0
463
+ total_params = 0
464
+ flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
465
+ for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
466
+ unpartitioned_numel = shape.numel()
467
+ total_numel += unpartitioned_numel
468
+ total_params += 1
469
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
470
+
471
+ if debug:
472
+ print(
473
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
474
+ )
475
+
476
+ # memory efficient tensor
477
+ tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
478
+ state_dict[name] = tensor
479
+ offset += partitioned_numel
480
+
481
+ offset *= world_size
482
+
483
+ # Sanity check
484
+ if offset != avail_numel:
485
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
486
+
487
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
488
+
489
+
490
+ def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
491
+ exclude_frozen_parameters):
492
+ state_dict = OrderedDict()
493
+
494
+ # buffers
495
+ buffers = zero_model_states[0].buffers
496
+ state_dict.update(buffers)
497
+ if debug:
498
+ print(f"added {len(buffers)} buffers")
499
+
500
+ if not exclude_frozen_parameters:
501
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
502
+
503
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
504
+
505
+ # recover shared parameters
506
+ for pair in zero_model_states[0].shared_params:
507
+ if pair[1] in state_dict:
508
+ state_dict[pair[0]] = state_dict[pair[1]]
509
+
510
+ return state_dict
511
+
512
+
513
+ def to_torch_tensor(state_dict, return_empty_tensor=False):
514
+ """
515
+ Convert state_dict of GatheredTensor to torch tensor
516
+ """
517
+ torch_state_dict = {}
518
+ converted_tensors = {}
519
+ for name, tensor in state_dict.items():
520
+ tensor_id = id(tensor)
521
+ if tensor_id in converted_tensors: # shared tensors
522
+ shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
523
+ torch_state_dict[name] = shared_tensor
524
+ else:
525
+ converted_tensors[tensor_id] = name
526
+ if return_empty_tensor:
527
+ torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
528
+ else:
529
+ torch_state_dict[name] = tensor.contiguous()
530
+ return torch_state_dict
531
+
532
+
533
+ def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
534
+ tag=None,
535
+ exclude_frozen_parameters=False,
536
+ lazy_mode=False):
537
+ """
538
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
539
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
540
+ via a model hub.
541
+
542
+ Args:
543
+ - ``checkpoint_dir``: path to the desired checkpoint folder
544
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
545
+ - ``exclude_frozen_parameters``: exclude frozen parameters
546
+ - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
547
+ Convert the pesduo tensor to torch tensor by ``.contiguous()``
548
+
549
+ Returns:
550
+ - pytorch ``state_dict``
551
+
552
+ A typical usage might be ::
553
+
554
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
555
+ # do the training and checkpoint saving
556
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
557
+ model = model.cpu() # move to cpu
558
+ model.load_state_dict(state_dict)
559
+ # submit to model hub or save the model to share with others
560
+
561
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
562
+ application. i.e. you will need to re-initialize the deepspeed engine, since
563
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
564
+
565
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
566
+
567
+ Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
568
+ You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
569
+ the checkpoint. Or you can load state_dict in lazy mode ::
570
+
571
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
572
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
573
+ for name, lazy_tensor in state_dict.item():
574
+ tensor = lazy_tensor.contiguous() # to cpu
575
+ print(name, tensor)
576
+ # del tensor to release memory if it no longer in use
577
+ """
578
+ if tag is None:
579
+ latest_path = os.path.join(checkpoint_dir, 'latest')
580
+ if os.path.isfile(latest_path):
581
+ with open(latest_path, 'r') as fd:
582
+ tag = fd.read().strip()
583
+ else:
584
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
585
+
586
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
587
+
588
+ if not os.path.isdir(ds_checkpoint_dir):
589
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
590
+
591
+ state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
592
+ if lazy_mode:
593
+ return state_dict
594
+ else:
595
+ return to_torch_tensor(state_dict)
596
+
597
+
598
+ def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
599
+ output_dir,
600
+ max_shard_size="5GB",
601
+ safe_serialization=False,
602
+ tag=None,
603
+ exclude_frozen_parameters=False):
604
+ """
605
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
606
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
607
+
608
+ Args:
609
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
610
+ - ``output_dir``: directory to the pytorch fp32 state_dict output files
611
+ - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
612
+ - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
613
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
614
+ - ``exclude_frozen_parameters``: exclude frozen parameters
615
+ """
616
+
617
+ # Dependency pre-check
618
+ if safe_serialization:
619
+ try:
620
+ from safetensors.torch import save_file
621
+ except ImportError:
622
+ print('If you want to use `safe_serialization`, please `pip install safetensors`')
623
+ raise
624
+ if max_shard_size is not None:
625
+ try:
626
+ from huggingface_hub import split_torch_state_dict_into_shards
627
+ except ImportError:
628
+ print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
629
+ raise
630
+
631
+ # Convert zero checkpoint to state_dict
632
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
633
+ tag,
634
+ exclude_frozen_parameters,
635
+ lazy_mode=True)
636
+
637
+ # Shard the model if it is too big.
638
+ weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
639
+ if max_shard_size is not None:
640
+ filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
641
+ # an memory-efficient approach for sharding
642
+ empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
643
+ state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
644
+ filename_pattern=filename_pattern,
645
+ max_shard_size=max_shard_size)
646
+ else:
647
+ from collections import namedtuple
648
+ StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
649
+ state_dict_split = StateDictSplit(is_sharded=False,
650
+ filename_to_tensors={weights_name: list(state_dict.keys())})
651
+
652
+ # Save the model by shard
653
+ os.makedirs(output_dir, exist_ok=True)
654
+ filename_to_tensors = state_dict_split.filename_to_tensors.items()
655
+ for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
656
+ shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
657
+ shard_state_dict = to_torch_tensor(shard_state_dict)
658
+ output_path = os.path.join(output_dir, shard_file)
659
+ if safe_serialization:
660
+ save_file(shard_state_dict, output_path, metadata={"format": "pt"})
661
+ else:
662
+ torch.save(shard_state_dict, output_path)
663
+ # release the memory of current shard
664
+ for tensor_name in list(shard_state_dict.keys()):
665
+ del state_dict[tensor_name]
666
+ del shard_state_dict[tensor_name]
667
+ del shard_state_dict
668
+ gc.collect()
669
+
670
+ # Save index if sharded
671
+ if state_dict_split.is_sharded:
672
+ index = {
673
+ "metadata": state_dict_split.metadata,
674
+ "weight_map": state_dict_split.tensor_to_filename,
675
+ }
676
+ save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
677
+ save_index_file = os.path.join(output_dir, save_index_file)
678
+ with open(save_index_file, "w", encoding="utf-8") as f:
679
+ content = json.dumps(index, indent=2, sort_keys=True) + "\n"
680
+ f.write(content)
681
+
682
+
683
+ def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
684
+ """
685
+ 1. Put the provided model to cpu
686
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
687
+ 3. Load it into the provided model
688
+
689
+ Args:
690
+ - ``model``: the model object to update
691
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
692
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
693
+
694
+ Returns:
695
+ - ``model`: modified model
696
+
697
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
698
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
699
+ conveniently placed for you in the checkpoint folder.
700
+
701
+ A typical usage might be ::
702
+
703
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
704
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
705
+ # submit to model hub or save the model to share with others
706
+
707
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
708
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
709
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
710
+
711
+ """
712
+ logger.info(f"Extracting fp32 weights")
713
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
714
+
715
+ logger.info(f"Overwriting model with fp32 weights")
716
+ model = model.cpu()
717
+ model.load_state_dict(state_dict, strict=False)
718
+
719
+ return model
720
+
721
+
722
+ if __name__ == "__main__":
723
+ parser = argparse.ArgumentParser()
724
+ parser.add_argument("checkpoint_dir",
725
+ type=str,
726
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
727
+ parser.add_argument("output_dir",
728
+ type=str,
729
+ help="directory to the pytorch fp32 state_dict output files"
730
+ "(e.g. path/checkpoint-12-output/)")
731
+ parser.add_argument(
732
+ "--max_shard_size",
733
+ type=str,
734
+ default="5GB",
735
+ help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
736
+ "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
737
+ "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
738
+ "without CPU OOM issues.")
739
+ parser.add_argument(
740
+ "--safe_serialization",
741
+ default=False,
742
+ action='store_true',
743
+ help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
744
+ parser.add_argument("-t",
745
+ "--tag",
746
+ type=str,
747
+ default=None,
748
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
749
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
750
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
751
+ args = parser.parse_args()
752
+
753
+ debug = args.debug
754
+
755
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
756
+ args.output_dir,
757
+ max_shard_size=args.max_shard_size,
758
+ safe_serialization=args.safe_serialization,
759
+ tag=args.tag,
760
+ exclude_frozen_parameters=args.exclude_frozen_parameters)
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/images/eval_loss.png ADDED
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/images/eval_runtime.png ADDED
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/images/eval_samples_per_second.png ADDED
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/images/eval_steps_per_second.png ADDED
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/images/eval_token_acc.png ADDED
qwen2.5-7b-2225q-206 q-1369q-newbs-new-click-2ep-lr1e-6/images/train_epoch.png ADDED