Hugging Face's logo

ChavyvAkvar
/
mla-baseline-1-epoch

Safetensors
mla
Model card Files
xet
Community
ChavyvAkvar commited on
Commit
eedebcd
·
verified ·
1 Parent(s): d6e19b6

Upload folder using huggingface_hub

Browse files
Files changed (29) hide show
  1. checkpoint-4000/config.json +37 -0
  2. checkpoint-4000/generation_config.json +6 -0
  3. checkpoint-4000/model.safetensors +3 -0
  4. checkpoint-4000/optimizer.pt +3 -0
  5. checkpoint-4000/rng_state.pth +3 -0
  6. checkpoint-4000/scheduler.pt +3 -0
  7. checkpoint-4000/special_tokens_map.json +28 -0
  8. checkpoint-4000/tokenizer.json +1085 -0
  9. checkpoint-4000/tokenizer_config.json +64 -0
  10. checkpoint-4000/trainer_state.json +594 -0
  11. checkpoint-4000/training_args.bin +3 -0
  12. checkpoint-5209/config.json +37 -0
  13. checkpoint-5209/generation_config.json +6 -0
  14. checkpoint-5209/model.safetensors +3 -0
  15. checkpoint-5209/optimizer.pt +3 -0
  16. checkpoint-5209/rng_state.pth +3 -0
  17. checkpoint-5209/scheduler.pt +3 -0
  18. checkpoint-5209/special_tokens_map.json +28 -0
  19. checkpoint-5209/tokenizer.json +1085 -0
  20. checkpoint-5209/tokenizer_config.json +64 -0
  21. checkpoint-5209/trainer_state.json +762 -0
  22. checkpoint-5209/training_args.bin +3 -0
  23. config.json +37 -0
  24. generation_config.json +6 -0
  25. model.safetensors +3 -0
  26. special_tokens_map.json +28 -0
  27. tokenizer.json +1085 -0
  28. tokenizer_config.json +64 -0
  29. training_args.bin +3 -0
checkpoint-4000/config.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "MLAForCausalLM"
4
+ ],
5
+ "bos_token_id": 0,
6
+ "dtype": "float32",
7
+ "elementwise_affine": true,
8
+ "eos_token_id": 1,
9
+ "fuse_cross_entropy": true,
10
+ "fuse_linear_cross_entropy": false,
11
+ "fuse_norm": true,
12
+ "fuse_swiglu": true,
13
+ "hidden_act": "swish",
14
+ "hidden_ratio": 4,
15
+ "hidden_size": 256,
16
+ "initializer_range": 0.02,
17
+ "intermediate_size": 896,
18
+ "kv_lora_rank": 64,
19
+ "max_position_embeddings": 2048,
20
+ "model_type": "mla",
21
+ "norm_eps": 1e-05,
22
+ "num_heads": 4,
23
+ "num_hidden_layers": 35,
24
+ "q_lora_rank": 32,
25
+ "qk_head_dim": 96,
26
+ "qk_nope_head_dim": 64,
27
+ "qk_rope_head_dim": 32,
28
+ "rope_scaling": null,
29
+ "rope_theta": 10000.0,
30
+ "tie_word_embeddings": true,
31
+ "transformers_version": "4.57.1",
32
+ "use_cache": true,
33
+ "use_l2warp": false,
34
+ "v_head_dim": 32,
35
+ "vocab_size": 256,
36
+ "window_size": null
37
+ }
checkpoint-4000/generation_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 0,
4
+ "eos_token_id": 1,
5
+ "transformers_version": "4.57.1"
6
+ }
checkpoint-4000/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cc06421d6b5ebd2d950235e0144b34004d13eb32a709a547dd7ee18888ff0a08
3
+ size 111070184
checkpoint-4000/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:79648d276f55fe05231992ee5c60f9b88b9b8b61ba8e8116090829966a5d3abd
3
+ size 222434622
checkpoint-4000/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4
3
+ size 14645
checkpoint-4000/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e3a35abd1700ed236b48f18fa90559a7c0b18459c8f5cea84dc18fc77113617f
3
+ size 1465
checkpoint-4000/special_tokens_map.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<think>",
4
+ "</think>"
5
+ ],
6
+ "bos_token": {
7
+ "content": "<|startoftext|>",
8
+ "lstrip": false,
9
+ "normalized": false,
10
+ "rstrip": false,
11
+ "single_word": false
12
+ },
13
+ "eos_token": {
14
+ "content": "<|endoftext|>",
15
+ "lstrip": false,
16
+ "normalized": false,
17
+ "rstrip": false,
18
+ "single_word": false
19
+ },
20
+ "pad_token": "<|pad|>",
21
+ "unk_token": {
22
+ "content": "<|unknown|>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false
27
+ }
28
+ }
checkpoint-4000/tokenizer.json ADDED
@@ -0,0 +1,1085 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "version": "1.0",
3
+ "truncation": null,
4
+ "padding": null,
5
+ "added_tokens": [
6
+ {
7
+ "id": 0,
8
+ "content": "<|startoftext|>",
9
+ "single_word": false,
10
+ "lstrip": false,
11
+ "rstrip": false,
12
+ "normalized": false,
13
+ "special": true
14
+ },
15
+ {
16
+ "id": 1,
17
+ "content": "<|endoftext|>",
18
+ "single_word": false,
19
+ "lstrip": false,
20
+ "rstrip": false,
21
+ "normalized": false,
22
+ "special": true
23
+ },
24
+ {
25
+ "id": 2,
26
+ "content": "<|unknown|>",
27
+ "single_word": false,
28
+ "lstrip": false,
29
+ "rstrip": false,
30
+ "normalized": false,
31
+ "special": true
32
+ },
33
+ {
34
+ "id": 3,
35
+ "content": "<|pad|>",
36
+ "single_word": false,
37
+ "lstrip": false,
38
+ "rstrip": false,
39
+ "normalized": false,
40
+ "special": true
41
+ },
42
+ {
43
+ "id": 4,
44
+ "content": "<think>",
45
+ "single_word": false,
46
+ "lstrip": false,
47
+ "rstrip": false,
48
+ "normalized": false,
49
+ "special": true
50
+ },
51
+ {
52
+ "id": 5,
53
+ "content": "</think>",
54
+ "single_word": false,
55
+ "lstrip": false,
56
+ "rstrip": false,
57
+ "normalized": false,
58
+ "special": true
59
+ }
60
+ ],
61
+ "normalizer": null,
62
+ "pre_tokenizer": {
63
+ "type": "ByteLevel",
64
+ "add_prefix_space": false,
65
+ "trim_offsets": true,
66
+ "use_regex": true
67
+ },
68
+ "post_processor": null,
69
+ "decoder": {
70
+ "type": "ByteLevel",
71
+ "add_prefix_space": true,
72
+ "trim_offsets": true,
73
+ "use_regex": true
74
+ },
75
+ "model": {
76
+ "type": "BPE",
77
+ "dropout": null,
78
+ "unk_token": null,
79
+ "continuing_subword_prefix": null,
80
+ "end_of_word_suffix": null,
81
+ "fuse_unk": false,
82
+ "byte_fallback": false,
83
+ "ignore_merges": false,
84
+ "vocab": {
85
+ "<|startoftext|>": 0,
86
+ "<|endoftext|>": 1,
87
+ "<|unknown|>": 2,
88
+ "<|pad|>": 3,
89
+ "<think>": 4,
90
+ "</think>": 5,
91
+ "%": 6,
92
+ "&": 7,
93
+ "'": 8,
94
+ "(": 9,
95
+ ")": 10,
96
+ ",": 11,
97
+ "-": 12,
98
+ ".": 13,
99
+ "0": 14,
100
+ "1": 15,
101
+ "2": 16,
102
+ "3": 17,
103
+ "4": 18,
104
+ "5": 19,
105
+ "6": 20,
106
+ "7": 21,
107
+ "8": 22,
108
+ "9": 23,
109
+ ":": 24,
110
+ ">": 25,
111
+ "A": 26,
112
+ "C": 27,
113
+ "D": 28,
114
+ "E": 29,
115
+ "F": 30,
116
+ "G": 31,
117
+ "H": 32,
118
+ "I": 33,
119
+ "L": 34,
120
+ "M": 35,
121
+ "N": 36,
122
+ "O": 37,
123
+ "P": 38,
124
+ "R": 39,
125
+ "S": 40,
126
+ "T": 41,
127
+ "V": 42,
128
+ "[": 43,
129
+ "]": 44,
130
+ "a": 45,
131
+ "b": 46,
132
+ "c": 47,
133
+ "d": 48,
134
+ "e": 49,
135
+ "f": 50,
136
+ "g": 51,
137
+ "h": 52,
138
+ "i": 53,
139
+ "k": 54,
140
+ "l": 55,
141
+ "m": 56,
142
+ "n": 57,
143
+ "o": 58,
144
+ "p": 59,
145
+ "q": 60,
146
+ "r": 61,
147
+ "s": 62,
148
+ "t": 63,
149
+ "u": 64,
150
+ "v": 65,
151
+ "x": 66,
152
+ "y": 67,
153
+ "|": 68,
154
+ "Ċ": 69,
155
+ "Ġ": 70,
156
+ "00": 71,
157
+ "11": 72,
158
+ "->": 73,
159
+ "Ġ|": 74,
160
+ ":[": 75,
161
+ "]->": 76,
162
+ "10": 77,
163
+ "Ġ1": 78,
164
+ "000": 79,
165
+ "ĠS": 80,
166
+ "',": 81,
167
+ "Ġ'": 82,
168
+ "te": 83,
169
+ "100": 84,
170
+ "ĊĠ": 85,
171
+ "110": 86,
172
+ "tep": 87,
173
+ "re": 88,
174
+ "111": 89,
175
+ "0000": 90,
176
+ "011": 91,
177
+ "101": 92,
178
+ "001": 93,
179
+ "ĠStep": 94,
180
+ "010": 95,
181
+ "in": 96,
182
+ "im": 97,
183
+ "ĠSim": 98,
184
+ "1111": 99,
185
+ "Ġ[": 100,
186
+ "or": 101,
187
+ "ĠI": 102,
188
+ "ho": 103,
189
+ "od": 104,
190
+ "hood": 105,
191
+ "Ġin": 106,
192
+ "Ġ11": 107,
193
+ "Ġ10": 108,
194
+ "ne": 109,
195
+ "Ġne": 110,
196
+ "']": 111,
197
+ "bor": 112,
198
+ "dx": 113,
199
+ "gh": 114,
200
+ "ire": 115,
201
+ "igh": 116,
202
+ "qu": 117,
203
+ "Ġre": 118,
204
+ "Ġ['": 119,
205
+ "ĠIdx": 120,
206
+ "Ġneigh": 121,
207
+ "borhood": 122,
208
+ "ires": 123,
209
+ "quires": 124,
210
+ "Ġrequires": 125,
211
+ "Ġneighborhood": 126,
212
+ "Step": 127,
213
+ "ut": 128,
214
+ "Ġ3": 129,
215
+ "Ġ2": 130,
216
+ "Ġ4": 131,
217
+ "Ġ5": 132,
218
+ "Ġ6": 133,
219
+ "Ġ9": 134,
220
+ "Ġ8": 135,
221
+ "Ġ7": 136,
222
+ "Ġ0": 137,
223
+ "ca": 138,
224
+ "Ġ12": 139,
225
+ "Ġ13": 140,
226
+ "1010": 141,
227
+ "Ġ14": 142,
228
+ "Out": 143,
229
+ "put": 144,
230
+ "ĠOut": 145,
231
+ "ĠSca": 146,
232
+ "ĠOutput": 147,
233
+ "ĠScan": 148,
234
+ "Ġ011": 149,
235
+ "ing": 150,
236
+ "ti": 151,
237
+ "1100": 152,
238
+ "00000000": 153,
239
+ "Ġ15": 154,
240
+ "Ġ000": 155,
241
+ "on": 156,
242
+ "Ġ110": 157,
243
+ "ul": 158,
244
+ "Ġ100": 159,
245
+ "at": 160,
246
+ "tion": 161,
247
+ "Ġ010": 162,
248
+ "1011": 163,
249
+ "ĠC": 164,
250
+ "Ġ00": 165,
251
+ "00011": 166,
252
+ "50": 167,
253
+ "er": 168,
254
+ "10000": 169,
255
+ "nd": 170,
256
+ "Ġ16": 171,
257
+ "ule": 172,
258
+ "ĠR": 173,
259
+ "ed": 174,
260
+ "li": 175,
261
+ "Ġ001": 176,
262
+ "Ġ101": 177,
263
+ "ate": 178,
264
+ "ĠRule": 179,
265
+ "1000": 180,
266
+ "fi": 181,
267
+ "10100": 182,
268
+ "as": 183,
269
+ "Ġ17": 184,
270
+ "ĠM": 185,
271
+ "Ġ1111": 186,
272
+ "Ver": 187,
273
+ "ch": 188,
274
+ "es": 189,
275
+ "ifi": 190,
276
+ "mp": 191,
277
+ "ting": 192,
278
+ "cation": 193,
279
+ "atch": 194,
280
+ "Verifi": 195,
281
+ "Verification": 196,
282
+ "..": 197,
283
+ "11011": 198,
284
+ "41": 199,
285
+ "and": 200,
286
+ "date": 201,
287
+ "idate": 202,
288
+ "ĠCand": 203,
289
+ "ĠCandidate": 204,
290
+ "58": 205,
291
+ "ating": 206,
292
+ "10011": 207,
293
+ "ction": 208,
294
+ "111000": 209,
295
+ "000000": 210,
296
+ "10110": 211,
297
+ "Ġ00000000": 212,
298
+ "is": 213,
299
+ "match": 214,
300
+ "ĠMis": 215,
301
+ "ĠMismatch": 216,
302
+ "111111": 217,
303
+ "Tas": 218,
304
+ "Task": 219,
305
+ "Ġ18": 220,
306
+ "Gre": 221,
307
+ "Pr": 222,
308
+ "ĠL": 223,
309
+ "ĠGre": 224,
310
+ "ĠPr": 225,
311
+ "edy": 226,
312
+ "ĠGreedy": 227,
313
+ "At": 228,
314
+ "Ex": 229,
315
+ "Tes": 230,
316
+ "ĠTes": 231,
317
+ "temp": 232,
318
+ "Attemp": 233,
319
+ "ĠTesting": 234,
320
+ "Attempt": 235,
321
+ "100000": 236,
322
+ "Ġs": 237,
323
+ "Ġ0000": 238,
324
+ "uction": 239,
325
+ "ĠSteps": 240,
326
+ "xt": 241,
327
+ "101111": 242,
328
+ "al": 243,
329
+ "cul": 244,
330
+ "eli": 245,
331
+ "ior": 246,
332
+ "keli": 247,
333
+ "ver": 248,
334
+ "cal": 249,
335
+ "culating": 250,
336
+ "iors": 251,
337
+ "kelihood": 252,
338
+ "Ġ19": 253,
339
+ "11111111": 254,
340
+ ").": 255
341
+ },
342
+ "merges": [
343
+ [
344
+ "0",
345
+ "0"
346
+ ],
347
+ [
348
+ "1",
349
+ "1"
350
+ ],
351
+ [
352
+ "-",
353
+ ">"
354
+ ],
355
+ [
356
+ "Ġ",
357
+ "|"
358
+ ],
359
+ [
360
+ ":",
361
+ "["
362
+ ],
363
+ [
364
+ "]",
365
+ "->"
366
+ ],
367
+ [
368
+ "1",
369
+ "0"
370
+ ],
371
+ [
372
+ "Ġ",
373
+ "1"
374
+ ],
375
+ [
376
+ "00",
377
+ "0"
378
+ ],
379
+ [
380
+ "Ġ",
381
+ "S"
382
+ ],
383
+ [
384
+ "'",
385
+ ","
386
+ ],
387
+ [
388
+ "Ġ",
389
+ "'"
390
+ ],
391
+ [
392
+ "t",
393
+ "e"
394
+ ],
395
+ [
396
+ "1",
397
+ "00"
398
+ ],
399
+ [
400
+ "Ċ",
401
+ "Ġ"
402
+ ],
403
+ [
404
+ "11",
405
+ "0"
406
+ ],
407
+ [
408
+ "te",
409
+ "p"
410
+ ],
411
+ [
412
+ "r",
413
+ "e"
414
+ ],
415
+ [
416
+ "11",
417
+ "1"
418
+ ],
419
+ [
420
+ "00",
421
+ "00"
422
+ ],
423
+ [
424
+ "0",
425
+ "11"
426
+ ],
427
+ [
428
+ "10",
429
+ "1"
430
+ ],
431
+ [
432
+ "00",
433
+ "1"
434
+ ],
435
+ [
436
+ "ĠS",
437
+ "tep"
438
+ ],
439
+ [
440
+ "0",
441
+ "10"
442
+ ],
443
+ [
444
+ "i",
445
+ "n"
446
+ ],
447
+ [
448
+ "i",
449
+ "m"
450
+ ],
451
+ [
452
+ "ĠS",
453
+ "im"
454
+ ],
455
+ [
456
+ "11",
457
+ "11"
458
+ ],
459
+ [
460
+ "Ġ",
461
+ "["
462
+ ],
463
+ [
464
+ "o",
465
+ "r"
466
+ ],
467
+ [
468
+ "Ġ",
469
+ "I"
470
+ ],
471
+ [
472
+ "h",
473
+ "o"
474
+ ],
475
+ [
476
+ "o",
477
+ "d"
478
+ ],
479
+ [
480
+ "ho",
481
+ "od"
482
+ ],
483
+ [
484
+ "Ġ",
485
+ "in"
486
+ ],
487
+ [
488
+ "Ġ",
489
+ "11"
490
+ ],
491
+ [
492
+ "Ġ",
493
+ "10"
494
+ ],
495
+ [
496
+ "n",
497
+ "e"
498
+ ],
499
+ [
500
+ "Ġ",
501
+ "ne"
502
+ ],
503
+ [
504
+ "'",
505
+ "]"
506
+ ],
507
+ [
508
+ "b",
509
+ "or"
510
+ ],
511
+ [
512
+ "d",
513
+ "x"
514
+ ],
515
+ [
516
+ "g",
517
+ "h"
518
+ ],
519
+ [
520
+ "i",
521
+ "re"
522
+ ],
523
+ [
524
+ "i",
525
+ "gh"
526
+ ],
527
+ [
528
+ "q",
529
+ "u"
530
+ ],
531
+ [
532
+ "Ġ",
533
+ "re"
534
+ ],
535
+ [
536
+ "Ġ[",
537
+ "'"
538
+ ],
539
+ [
540
+ "ĠI",
541
+ "dx"
542
+ ],
543
+ [
544
+ "Ġne",
545
+ "igh"
546
+ ],
547
+ [
548
+ "bor",
549
+ "hood"
550
+ ],
551
+ [
552
+ "ire",
553
+ "s"
554
+ ],
555
+ [
556
+ "qu",
557
+ "ires"
558
+ ],
559
+ [
560
+ "Ġre",
561
+ "quires"
562
+ ],
563
+ [
564
+ "Ġneigh",
565
+ "borhood"
566
+ ],
567
+ [
568
+ "S",
569
+ "tep"
570
+ ],
571
+ [
572
+ "u",
573
+ "t"
574
+ ],
575
+ [
576
+ "Ġ",
577
+ "3"
578
+ ],
579
+ [
580
+ "Ġ",
581
+ "2"
582
+ ],
583
+ [
584
+ "Ġ",
585
+ "4"
586
+ ],
587
+ [
588
+ "Ġ",
589
+ "5"
590
+ ],
591
+ [
592
+ "Ġ",
593
+ "6"
594
+ ],
595
+ [
596
+ "Ġ",
597
+ "9"
598
+ ],
599
+ [
600
+ "Ġ",
601
+ "8"
602
+ ],
603
+ [
604
+ "Ġ",
605
+ "7"
606
+ ],
607
+ [
608
+ "Ġ",
609
+ "0"
610
+ ],
611
+ [
612
+ "c",
613
+ "a"
614
+ ],
615
+ [
616
+ "Ġ1",
617
+ "2"
618
+ ],
619
+ [
620
+ "Ġ1",
621
+ "3"
622
+ ],
623
+ [
624
+ "10",
625
+ "10"
626
+ ],
627
+ [
628
+ "Ġ1",
629
+ "4"
630
+ ],
631
+ [
632
+ "O",
633
+ "ut"
634
+ ],
635
+ [
636
+ "p",
637
+ "ut"
638
+ ],
639
+ [
640
+ "Ġ",
641
+ "Out"
642
+ ],
643
+ [
644
+ "ĠS",
645
+ "ca"
646
+ ],
647
+ [
648
+ "ĠOut",
649
+ "put"
650
+ ],
651
+ [
652
+ "ĠSca",
653
+ "n"
654
+ ],
655
+ [
656
+ "Ġ",
657
+ "011"
658
+ ],
659
+ [
660
+ "in",
661
+ "g"
662
+ ],
663
+ [
664
+ "t",
665
+ "i"
666
+ ],
667
+ [
668
+ "11",
669
+ "00"
670
+ ],
671
+ [
672
+ "0000",
673
+ "0000"
674
+ ],
675
+ [
676
+ "Ġ1",
677
+ "5"
678
+ ],
679
+ [
680
+ "Ġ",
681
+ "000"
682
+ ],
683
+ [
684
+ "o",
685
+ "n"
686
+ ],
687
+ [
688
+ "Ġ",
689
+ "110"
690
+ ],
691
+ [
692
+ "u",
693
+ "l"
694
+ ],
695
+ [
696
+ "Ġ1",
697
+ "00"
698
+ ],
699
+ [
700
+ "a",
701
+ "t"
702
+ ],
703
+ [
704
+ "ti",
705
+ "on"
706
+ ],
707
+ [
708
+ "Ġ",
709
+ "010"
710
+ ],
711
+ [
712
+ "10",
713
+ "11"
714
+ ],
715
+ [
716
+ "Ġ",
717
+ "C"
718
+ ],
719
+ [
720
+ "Ġ",
721
+ "00"
722
+ ],
723
+ [
724
+ "000",
725
+ "11"
726
+ ],
727
+ [
728
+ "5",
729
+ "0"
730
+ ],
731
+ [
732
+ "e",
733
+ "r"
734
+ ],
735
+ [
736
+ "100",
737
+ "00"
738
+ ],
739
+ [
740
+ "n",
741
+ "d"
742
+ ],
743
+ [
744
+ "Ġ1",
745
+ "6"
746
+ ],
747
+ [
748
+ "ul",
749
+ "e"
750
+ ],
751
+ [
752
+ "Ġ",
753
+ "R"
754
+ ],
755
+ [
756
+ "e",
757
+ "d"
758
+ ],
759
+ [
760
+ "l",
761
+ "i"
762
+ ],
763
+ [
764
+ "Ġ",
765
+ "001"
766
+ ],
767
+ [
768
+ "Ġ",
769
+ "101"
770
+ ],
771
+ [
772
+ "a",
773
+ "te"
774
+ ],
775
+ [
776
+ "ĠR",
777
+ "ule"
778
+ ],
779
+ [
780
+ "1",
781
+ "000"
782
+ ],
783
+ [
784
+ "f",
785
+ "i"
786
+ ],
787
+ [
788
+ "10",
789
+ "100"
790
+ ],
791
+ [
792
+ "a",
793
+ "s"
794
+ ],
795
+ [
796
+ "Ġ1",
797
+ "7"
798
+ ],
799
+ [
800
+ "Ġ",
801
+ "M"
802
+ ],
803
+ [
804
+ "Ġ",
805
+ "1111"
806
+ ],
807
+ [
808
+ "V",
809
+ "er"
810
+ ],
811
+ [
812
+ "c",
813
+ "h"
814
+ ],
815
+ [
816
+ "e",
817
+ "s"
818
+ ],
819
+ [
820
+ "i",
821
+ "fi"
822
+ ],
823
+ [
824
+ "m",
825
+ "p"
826
+ ],
827
+ [
828
+ "t",
829
+ "ing"
830
+ ],
831
+ [
832
+ "ca",
833
+ "tion"
834
+ ],
835
+ [
836
+ "at",
837
+ "ch"
838
+ ],
839
+ [
840
+ "Ver",
841
+ "ifi"
842
+ ],
843
+ [
844
+ "Verifi",
845
+ "cation"
846
+ ],
847
+ [
848
+ ".",
849
+ "."
850
+ ],
851
+ [
852
+ "110",
853
+ "11"
854
+ ],
855
+ [
856
+ "4",
857
+ "1"
858
+ ],
859
+ [
860
+ "a",
861
+ "nd"
862
+ ],
863
+ [
864
+ "d",
865
+ "ate"
866
+ ],
867
+ [
868
+ "i",
869
+ "date"
870
+ ],
871
+ [
872
+ "ĠC",
873
+ "and"
874
+ ],
875
+ [
876
+ "ĠCand",
877
+ "idate"
878
+ ],
879
+ [
880
+ "5",
881
+ "8"
882
+ ],
883
+ [
884
+ "at",
885
+ "ing"
886
+ ],
887
+ [
888
+ "100",
889
+ "11"
890
+ ],
891
+ [
892
+ "c",
893
+ "tion"
894
+ ],
895
+ [
896
+ "111",
897
+ "000"
898
+ ],
899
+ [
900
+ "0000",
901
+ "00"
902
+ ],
903
+ [
904
+ "10",
905
+ "110"
906
+ ],
907
+ [
908
+ "Ġ",
909
+ "00000000"
910
+ ],
911
+ [
912
+ "i",
913
+ "s"
914
+ ],
915
+ [
916
+ "m",
917
+ "atch"
918
+ ],
919
+ [
920
+ "ĠM",
921
+ "is"
922
+ ],
923
+ [
924
+ "ĠMis",
925
+ "match"
926
+ ],
927
+ [
928
+ "1111",
929
+ "11"
930
+ ],
931
+ [
932
+ "T",
933
+ "as"
934
+ ],
935
+ [
936
+ "Tas",
937
+ "k"
938
+ ],
939
+ [
940
+ "Ġ1",
941
+ "8"
942
+ ],
943
+ [
944
+ "G",
945
+ "re"
946
+ ],
947
+ [
948
+ "P",
949
+ "r"
950
+ ],
951
+ [
952
+ "Ġ",
953
+ "L"
954
+ ],
955
+ [
956
+ "Ġ",
957
+ "Gre"
958
+ ],
959
+ [
960
+ "Ġ",
961
+ "Pr"
962
+ ],
963
+ [
964
+ "ed",
965
+ "y"
966
+ ],
967
+ [
968
+ "ĠGre",
969
+ "edy"
970
+ ],
971
+ [
972
+ "A",
973
+ "t"
974
+ ],
975
+ [
976
+ "E",
977
+ "x"
978
+ ],
979
+ [
980
+ "T",
981
+ "es"
982
+ ],
983
+ [
984
+ "Ġ",
985
+ "Tes"
986
+ ],
987
+ [
988
+ "te",
989
+ "mp"
990
+ ],
991
+ [
992
+ "At",
993
+ "temp"
994
+ ],
995
+ [
996
+ "ĠTes",
997
+ "ting"
998
+ ],
999
+ [
1000
+ "Attemp",
1001
+ "t"
1002
+ ],
1003
+ [
1004
+ "100",
1005
+ "000"
1006
+ ],
1007
+ [
1008
+ "Ġ",
1009
+ "s"
1010
+ ],
1011
+ [
1012
+ "Ġ",
1013
+ "0000"
1014
+ ],
1015
+ [
1016
+ "u",
1017
+ "ction"
1018
+ ],
1019
+ [
1020
+ "ĠStep",
1021
+ "s"
1022
+ ],
1023
+ [
1024
+ "x",
1025
+ "t"
1026
+ ],
1027
+ [
1028
+ "10",
1029
+ "1111"
1030
+ ],
1031
+ [
1032
+ "a",
1033
+ "l"
1034
+ ],
1035
+ [
1036
+ "c",
1037
+ "ul"
1038
+ ],
1039
+ [
1040
+ "e",
1041
+ "li"
1042
+ ],
1043
+ [
1044
+ "i",
1045
+ "or"
1046
+ ],
1047
+ [
1048
+ "k",
1049
+ "eli"
1050
+ ],
1051
+ [
1052
+ "v",
1053
+ "er"
1054
+ ],
1055
+ [
1056
+ "ca",
1057
+ "l"
1058
+ ],
1059
+ [
1060
+ "cul",
1061
+ "ating"
1062
+ ],
1063
+ [
1064
+ "ior",
1065
+ "s"
1066
+ ],
1067
+ [
1068
+ "keli",
1069
+ "hood"
1070
+ ],
1071
+ [
1072
+ "Ġ1",
1073
+ "9"
1074
+ ],
1075
+ [
1076
+ "1111",
1077
+ "1111"
1078
+ ],
1079
+ [
1080
+ ")",
1081
+ "."
1082
+ ]
1083
+ ]
1084
+ }
1085
+ }
checkpoint-4000/tokenizer_config.json ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<|startoftext|>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "<|endoftext|>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "<|unknown|>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "3": {
28
+ "content": "<|pad|>",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "4": {
36
+ "content": "<think>",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ },
43
+ "5": {
44
+ "content": "</think>",
45
+ "lstrip": false,
46
+ "normalized": false,
47
+ "rstrip": false,
48
+ "single_word": false,
49
+ "special": true
50
+ }
51
+ },
52
+ "additional_special_tokens": [
53
+ "<think>",
54
+ "</think>"
55
+ ],
56
+ "bos_token": "<|startoftext|>",
57
+ "clean_up_tokenization_spaces": false,
58
+ "eos_token": "<|endoftext|>",
59
+ "extra_special_tokens": {},
60
+ "model_max_length": 1000000000000000019884624838656,
61
+ "pad_token": "<|pad|>",
62
+ "tokenizer_class": "PreTrainedTokenizerFast",
63
+ "unk_token": "<|unknown|>"
64
+ }
checkpoint-4000/trainer_state.json ADDED
@@ -0,0 +1,594 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 0.7679754247864068,
6
+ "eval_steps": 500,
7
+ "global_step": 4000,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.009599692809830085,
14
+ "grad_norm": 14.53178596496582,
15
+ "learning_rate": 1.47e-05,
16
+ "loss": 18.117,
17
+ "step": 50
18
+ },
19
+ {
20
+ "epoch": 0.01919938561966017,
21
+ "grad_norm": 8.379057884216309,
22
+ "learning_rate": 2.97e-05,
23
+ "loss": 11.1278,
24
+ "step": 100
25
+ },
26
+ {
27
+ "epoch": 0.028799078429490255,
28
+ "grad_norm": 17.656112670898438,
29
+ "learning_rate": 4.4699999999999996e-05,
30
+ "loss": 8.8796,
31
+ "step": 150
32
+ },
33
+ {
34
+ "epoch": 0.03839877123932034,
35
+ "grad_norm": 14.292523384094238,
36
+ "learning_rate": 5.97e-05,
37
+ "loss": 7.8564,
38
+ "step": 200
39
+ },
40
+ {
41
+ "epoch": 0.047998464049150424,
42
+ "grad_norm": 19.745756149291992,
43
+ "learning_rate": 7.47e-05,
44
+ "loss": 7.1001,
45
+ "step": 250
46
+ },
47
+ {
48
+ "epoch": 0.05759815685898051,
49
+ "grad_norm": 45.06932830810547,
50
+ "learning_rate": 8.969999999999998e-05,
51
+ "loss": 6.4644,
52
+ "step": 300
53
+ },
54
+ {
55
+ "epoch": 0.0671978496688106,
56
+ "grad_norm": 26.414615631103516,
57
+ "learning_rate": 0.00010469999999999998,
58
+ "loss": 5.9291,
59
+ "step": 350
60
+ },
61
+ {
62
+ "epoch": 0.07679754247864068,
63
+ "grad_norm": 21.97804069519043,
64
+ "learning_rate": 0.0001197,
65
+ "loss": 5.367,
66
+ "step": 400
67
+ },
68
+ {
69
+ "epoch": 0.08639723528847076,
70
+ "grad_norm": 21.083677291870117,
71
+ "learning_rate": 0.0001347,
72
+ "loss": 4.7741,
73
+ "step": 450
74
+ },
75
+ {
76
+ "epoch": 0.09599692809830085,
77
+ "grad_norm": 34.146934509277344,
78
+ "learning_rate": 0.00014969999999999998,
79
+ "loss": 4.3934,
80
+ "step": 500
81
+ },
82
+ {
83
+ "epoch": 0.10559662090813093,
84
+ "grad_norm": 31.148387908935547,
85
+ "learning_rate": 0.0001647,
86
+ "loss": 4.008,
87
+ "step": 550
88
+ },
89
+ {
90
+ "epoch": 0.11519631371796102,
91
+ "grad_norm": 21.12940788269043,
92
+ "learning_rate": 0.00017969999999999998,
93
+ "loss": 3.531,
94
+ "step": 600
95
+ },
96
+ {
97
+ "epoch": 0.12479600652779112,
98
+ "grad_norm": 13.038416862487793,
99
+ "learning_rate": 0.0001947,
100
+ "loss": 2.9393,
101
+ "step": 650
102
+ },
103
+ {
104
+ "epoch": 0.1343956993376212,
105
+ "grad_norm": 6.97022819519043,
106
+ "learning_rate": 0.00020969999999999997,
107
+ "loss": 2.4658,
108
+ "step": 700
109
+ },
110
+ {
111
+ "epoch": 0.14399539214745127,
112
+ "grad_norm": 11.445544242858887,
113
+ "learning_rate": 0.0002247,
114
+ "loss": 2.1561,
115
+ "step": 750
116
+ },
117
+ {
118
+ "epoch": 0.15359508495728136,
119
+ "grad_norm": 9.814635276794434,
120
+ "learning_rate": 0.0002397,
121
+ "loss": 1.8553,
122
+ "step": 800
123
+ },
124
+ {
125
+ "epoch": 0.16319477776711144,
126
+ "grad_norm": 10.05532169342041,
127
+ "learning_rate": 0.00025469999999999996,
128
+ "loss": 1.6969,
129
+ "step": 850
130
+ },
131
+ {
132
+ "epoch": 0.17279447057694153,
133
+ "grad_norm": 6.568735122680664,
134
+ "learning_rate": 0.0002697,
135
+ "loss": 1.5864,
136
+ "step": 900
137
+ },
138
+ {
139
+ "epoch": 0.1823941633867716,
140
+ "grad_norm": 11.432738304138184,
141
+ "learning_rate": 0.0002847,
142
+ "loss": 1.4549,
143
+ "step": 950
144
+ },
145
+ {
146
+ "epoch": 0.1919938561966017,
147
+ "grad_norm": 7.344091415405273,
148
+ "learning_rate": 0.00029969999999999997,
149
+ "loss": 1.3058,
150
+ "step": 1000
151
+ },
152
+ {
153
+ "epoch": 0.20159354900643178,
154
+ "grad_norm": 7.833723068237305,
155
+ "learning_rate": 0.0002965074839629365,
156
+ "loss": 1.2029,
157
+ "step": 1050
158
+ },
159
+ {
160
+ "epoch": 0.21119324181626187,
161
+ "grad_norm": 9.809481620788574,
162
+ "learning_rate": 0.000292943692088382,
163
+ "loss": 1.1299,
164
+ "step": 1100
165
+ },
166
+ {
167
+ "epoch": 0.22079293462609195,
168
+ "grad_norm": 10.855881690979004,
169
+ "learning_rate": 0.0002893799002138275,
170
+ "loss": 1.0214,
171
+ "step": 1150
172
+ },
173
+ {
174
+ "epoch": 0.23039262743592204,
175
+ "grad_norm": 2.817777156829834,
176
+ "learning_rate": 0.000285816108339273,
177
+ "loss": 0.9556,
178
+ "step": 1200
179
+ },
180
+ {
181
+ "epoch": 0.23999232024575212,
182
+ "grad_norm": 5.34994649887085,
183
+ "learning_rate": 0.00028225231646471845,
184
+ "loss": 0.8728,
185
+ "step": 1250
186
+ },
187
+ {
188
+ "epoch": 0.24959201305558223,
189
+ "grad_norm": 1.8884717226028442,
190
+ "learning_rate": 0.0002786885245901639,
191
+ "loss": 0.7461,
192
+ "step": 1300
193
+ },
194
+ {
195
+ "epoch": 0.2591917058654123,
196
+ "grad_norm": 1.8414419889450073,
197
+ "learning_rate": 0.0002751247327156094,
198
+ "loss": 0.6367,
199
+ "step": 1350
200
+ },
201
+ {
202
+ "epoch": 0.2687913986752424,
203
+ "grad_norm": 1.6845418214797974,
204
+ "learning_rate": 0.00027156094084105487,
205
+ "loss": 0.5393,
206
+ "step": 1400
207
+ },
208
+ {
209
+ "epoch": 0.27839109148507246,
210
+ "grad_norm": 0.8279030323028564,
211
+ "learning_rate": 0.00026799714896650034,
212
+ "loss": 0.4878,
213
+ "step": 1450
214
+ },
215
+ {
216
+ "epoch": 0.28799078429490255,
217
+ "grad_norm": 1.0994820594787598,
218
+ "learning_rate": 0.0002644333570919458,
219
+ "loss": 0.4248,
220
+ "step": 1500
221
+ },
222
+ {
223
+ "epoch": 0.29759047710473263,
224
+ "grad_norm": 1.264716625213623,
225
+ "learning_rate": 0.0002608695652173913,
226
+ "loss": 0.3817,
227
+ "step": 1550
228
+ },
229
+ {
230
+ "epoch": 0.3071901699145627,
231
+ "grad_norm": 0.8758682012557983,
232
+ "learning_rate": 0.00025730577334283675,
233
+ "loss": 0.3623,
234
+ "step": 1600
235
+ },
236
+ {
237
+ "epoch": 0.3167898627243928,
238
+ "grad_norm": 0.5896995663642883,
239
+ "learning_rate": 0.0002537419814682822,
240
+ "loss": 0.3208,
241
+ "step": 1650
242
+ },
243
+ {
244
+ "epoch": 0.3263895555342229,
245
+ "grad_norm": 0.7703513503074646,
246
+ "learning_rate": 0.0002501781895937277,
247
+ "loss": 0.2889,
248
+ "step": 1700
249
+ },
250
+ {
251
+ "epoch": 0.33598924834405297,
252
+ "grad_norm": 0.5982457399368286,
253
+ "learning_rate": 0.00024661439771917317,
254
+ "loss": 0.2803,
255
+ "step": 1750
256
+ },
257
+ {
258
+ "epoch": 0.34558894115388306,
259
+ "grad_norm": 0.5336303114891052,
260
+ "learning_rate": 0.00024305060584461867,
261
+ "loss": 0.3014,
262
+ "step": 1800
263
+ },
264
+ {
265
+ "epoch": 0.35518863396371314,
266
+ "grad_norm": 1.1344468593597412,
267
+ "learning_rate": 0.00023948681397006414,
268
+ "loss": 0.2615,
269
+ "step": 1850
270
+ },
271
+ {
272
+ "epoch": 0.3647883267735432,
273
+ "grad_norm": 1.2401304244995117,
274
+ "learning_rate": 0.0002359230220955096,
275
+ "loss": 0.2436,
276
+ "step": 1900
277
+ },
278
+ {
279
+ "epoch": 0.3743880195833733,
280
+ "grad_norm": 0.8760889172554016,
281
+ "learning_rate": 0.00023235923022095505,
282
+ "loss": 0.221,
283
+ "step": 1950
284
+ },
285
+ {
286
+ "epoch": 0.3839877123932034,
287
+ "grad_norm": 0.6058375239372253,
288
+ "learning_rate": 0.00022879543834640053,
289
+ "loss": 0.2061,
290
+ "step": 2000
291
+ },
292
+ {
293
+ "epoch": 0.3935874052030335,
294
+ "grad_norm": 0.5243856906890869,
295
+ "learning_rate": 0.00022523164647184602,
296
+ "loss": 0.1934,
297
+ "step": 2050
298
+ },
299
+ {
300
+ "epoch": 0.40318709801286357,
301
+ "grad_norm": 0.45033150911331177,
302
+ "learning_rate": 0.0002216678545972915,
303
+ "loss": 0.1869,
304
+ "step": 2100
305
+ },
306
+ {
307
+ "epoch": 0.41278679082269365,
308
+ "grad_norm": 0.5008948445320129,
309
+ "learning_rate": 0.00021810406272273697,
310
+ "loss": 0.1843,
311
+ "step": 2150
312
+ },
313
+ {
314
+ "epoch": 0.42238648363252373,
315
+ "grad_norm": 1.0652313232421875,
316
+ "learning_rate": 0.00021454027084818244,
317
+ "loss": 0.1811,
318
+ "step": 2200
319
+ },
320
+ {
321
+ "epoch": 0.4319861764423538,
322
+ "grad_norm": 0.3990040421485901,
323
+ "learning_rate": 0.00021097647897362794,
324
+ "loss": 0.1791,
325
+ "step": 2250
326
+ },
327
+ {
328
+ "epoch": 0.4415858692521839,
329
+ "grad_norm": 0.4828120172023773,
330
+ "learning_rate": 0.0002074126870990734,
331
+ "loss": 0.1681,
332
+ "step": 2300
333
+ },
334
+ {
335
+ "epoch": 0.451185562062014,
336
+ "grad_norm": 0.5252107977867126,
337
+ "learning_rate": 0.00020384889522451888,
338
+ "loss": 0.164,
339
+ "step": 2350
340
+ },
341
+ {
342
+ "epoch": 0.4607852548718441,
343
+ "grad_norm": 0.24024103581905365,
344
+ "learning_rate": 0.00020028510334996435,
345
+ "loss": 0.1518,
346
+ "step": 2400
347
+ },
348
+ {
349
+ "epoch": 0.47038494768167416,
350
+ "grad_norm": 0.5373029708862305,
351
+ "learning_rate": 0.0001967213114754098,
352
+ "loss": 0.1572,
353
+ "step": 2450
354
+ },
355
+ {
356
+ "epoch": 0.47998464049150424,
357
+ "grad_norm": 0.41635262966156006,
358
+ "learning_rate": 0.00019315751960085527,
359
+ "loss": 0.1459,
360
+ "step": 2500
361
+ },
362
+ {
363
+ "epoch": 0.4895843333013344,
364
+ "grad_norm": 0.26312512159347534,
365
+ "learning_rate": 0.00018959372772630077,
366
+ "loss": 0.1444,
367
+ "step": 2550
368
+ },
369
+ {
370
+ "epoch": 0.49918402611116447,
371
+ "grad_norm": 0.33238592743873596,
372
+ "learning_rate": 0.00018602993585174624,
373
+ "loss": 0.1415,
374
+ "step": 2600
375
+ },
376
+ {
377
+ "epoch": 0.5087837189209945,
378
+ "grad_norm": 0.4475176930427551,
379
+ "learning_rate": 0.0001824661439771917,
380
+ "loss": 0.1446,
381
+ "step": 2650
382
+ },
383
+ {
384
+ "epoch": 0.5183834117308246,
385
+ "grad_norm": 0.4537954032421112,
386
+ "learning_rate": 0.00017890235210263718,
387
+ "loss": 0.1395,
388
+ "step": 2700
389
+ },
390
+ {
391
+ "epoch": 0.5279831045406547,
392
+ "grad_norm": 0.3630366921424866,
393
+ "learning_rate": 0.00017533856022808268,
394
+ "loss": 0.1371,
395
+ "step": 2750
396
+ },
397
+ {
398
+ "epoch": 0.5375827973504848,
399
+ "grad_norm": 0.44030898809432983,
400
+ "learning_rate": 0.00017177476835352815,
401
+ "loss": 0.1377,
402
+ "step": 2800
403
+ },
404
+ {
405
+ "epoch": 0.5471824901603148,
406
+ "grad_norm": 0.4108527898788452,
407
+ "learning_rate": 0.00016821097647897363,
408
+ "loss": 0.1331,
409
+ "step": 2850
410
+ },
411
+ {
412
+ "epoch": 0.5567821829701449,
413
+ "grad_norm": 0.45702821016311646,
414
+ "learning_rate": 0.0001646471846044191,
415
+ "loss": 0.1325,
416
+ "step": 2900
417
+ },
418
+ {
419
+ "epoch": 0.566381875779975,
420
+ "grad_norm": 0.46175822615623474,
421
+ "learning_rate": 0.00016108339272986454,
422
+ "loss": 0.1273,
423
+ "step": 2950
424
+ },
425
+ {
426
+ "epoch": 0.5759815685898051,
427
+ "grad_norm": 0.41439440846443176,
428
+ "learning_rate": 0.00015751960085531,
429
+ "loss": 0.1309,
430
+ "step": 3000
431
+ },
432
+ {
433
+ "epoch": 0.5855812613996352,
434
+ "grad_norm": 0.3587839901447296,
435
+ "learning_rate": 0.0001539558089807555,
436
+ "loss": 0.1242,
437
+ "step": 3050
438
+ },
439
+ {
440
+ "epoch": 0.5951809542094653,
441
+ "grad_norm": 0.42960047721862793,
442
+ "learning_rate": 0.00015039201710620098,
443
+ "loss": 0.127,
444
+ "step": 3100
445
+ },
446
+ {
447
+ "epoch": 0.6047806470192953,
448
+ "grad_norm": 0.2696012854576111,
449
+ "learning_rate": 0.00014682822523164645,
450
+ "loss": 0.1253,
451
+ "step": 3150
452
+ },
453
+ {
454
+ "epoch": 0.6143803398291254,
455
+ "grad_norm": 0.2405436486005783,
456
+ "learning_rate": 0.00014326443335709193,
457
+ "loss": 0.121,
458
+ "step": 3200
459
+ },
460
+ {
461
+ "epoch": 0.6239800326389555,
462
+ "grad_norm": 0.20173802971839905,
463
+ "learning_rate": 0.00013970064148253743,
464
+ "loss": 0.1211,
465
+ "step": 3250
466
+ },
467
+ {
468
+ "epoch": 0.6335797254487856,
469
+ "grad_norm": 0.37551426887512207,
470
+ "learning_rate": 0.0001361368496079829,
471
+ "loss": 0.1189,
472
+ "step": 3300
473
+ },
474
+ {
475
+ "epoch": 0.6431794182586157,
476
+ "grad_norm": 0.2555278539657593,
477
+ "learning_rate": 0.00013257305773342834,
478
+ "loss": 0.1236,
479
+ "step": 3350
480
+ },
481
+ {
482
+ "epoch": 0.6527791110684458,
483
+ "grad_norm": 0.2080872356891632,
484
+ "learning_rate": 0.00012900926585887384,
485
+ "loss": 0.1187,
486
+ "step": 3400
487
+ },
488
+ {
489
+ "epoch": 0.6623788038782759,
490
+ "grad_norm": 0.27079683542251587,
491
+ "learning_rate": 0.0001254454739843193,
492
+ "loss": 0.1169,
493
+ "step": 3450
494
+ },
495
+ {
496
+ "epoch": 0.6719784966881059,
497
+ "grad_norm": 0.3163507282733917,
498
+ "learning_rate": 0.00012188168210976478,
499
+ "loss": 0.1182,
500
+ "step": 3500
501
+ },
502
+ {
503
+ "epoch": 0.681578189497936,
504
+ "grad_norm": 0.23527826368808746,
505
+ "learning_rate": 0.00011831789023521026,
506
+ "loss": 0.1144,
507
+ "step": 3550
508
+ },
509
+ {
510
+ "epoch": 0.6911778823077661,
511
+ "grad_norm": 0.2670173943042755,
512
+ "learning_rate": 0.00011475409836065573,
513
+ "loss": 0.1133,
514
+ "step": 3600
515
+ },
516
+ {
517
+ "epoch": 0.7007775751175962,
518
+ "grad_norm": 0.22517059743404388,
519
+ "learning_rate": 0.0001111903064861012,
520
+ "loss": 0.1153,
521
+ "step": 3650
522
+ },
523
+ {
524
+ "epoch": 0.7103772679274263,
525
+ "grad_norm": 0.22424179315567017,
526
+ "learning_rate": 0.00010762651461154668,
527
+ "loss": 0.1125,
528
+ "step": 3700
529
+ },
530
+ {
531
+ "epoch": 0.7199769607372564,
532
+ "grad_norm": 0.24241302907466888,
533
+ "learning_rate": 0.00010406272273699216,
534
+ "loss": 0.1132,
535
+ "step": 3750
536
+ },
537
+ {
538
+ "epoch": 0.7295766535470865,
539
+ "grad_norm": 0.3849368393421173,
540
+ "learning_rate": 0.00010049893086243763,
541
+ "loss": 0.1129,
542
+ "step": 3800
543
+ },
544
+ {
545
+ "epoch": 0.7391763463569165,
546
+ "grad_norm": 0.26350146532058716,
547
+ "learning_rate": 9.69351389878831e-05,
548
+ "loss": 0.1098,
549
+ "step": 3850
550
+ },
551
+ {
552
+ "epoch": 0.7487760391667466,
553
+ "grad_norm": 0.15014544129371643,
554
+ "learning_rate": 9.337134711332857e-05,
555
+ "loss": 0.1083,
556
+ "step": 3900
557
+ },
558
+ {
559
+ "epoch": 0.7583757319765767,
560
+ "grad_norm": 0.23375675082206726,
561
+ "learning_rate": 8.980755523877404e-05,
562
+ "loss": 0.1085,
563
+ "step": 3950
564
+ },
565
+ {
566
+ "epoch": 0.7679754247864068,
567
+ "grad_norm": 0.19782401621341705,
568
+ "learning_rate": 8.624376336421953e-05,
569
+ "loss": 0.1088,
570
+ "step": 4000
571
+ }
572
+ ],
573
+ "logging_steps": 50,
574
+ "max_steps": 5209,
575
+ "num_input_tokens_seen": 0,
576
+ "num_train_epochs": 1,
577
+ "save_steps": 2000,
578
+ "stateful_callbacks": {
579
+ "TrainerControl": {
580
+ "args": {
581
+ "should_epoch_stop": false,
582
+ "should_evaluate": false,
583
+ "should_log": false,
584
+ "should_save": true,
585
+ "should_training_stop": false
586
+ },
587
+ "attributes": {}
588
+ }
589
+ },
590
+ "total_flos": 8.7105258651648e+16,
591
+ "train_batch_size": 16,
592
+ "trial_name": null,
593
+ "trial_params": null
594
+ }
checkpoint-4000/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:30878ac8e72b88c8cbfce7348f711b9b45f86117719d802abb44671778afc8df
3
+ size 5841
checkpoint-5209/config.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "MLAForCausalLM"
4
+ ],
5
+ "bos_token_id": 0,
6
+ "dtype": "float32",
7
+ "elementwise_affine": true,
8
+ "eos_token_id": 1,
9
+ "fuse_cross_entropy": true,
10
+ "fuse_linear_cross_entropy": false,
11
+ "fuse_norm": true,
12
+ "fuse_swiglu": true,
13
+ "hidden_act": "swish",
14
+ "hidden_ratio": 4,
15
+ "hidden_size": 256,
16
+ "initializer_range": 0.02,
17
+ "intermediate_size": 896,
18
+ "kv_lora_rank": 64,
19
+ "max_position_embeddings": 2048,
20
+ "model_type": "mla",
21
+ "norm_eps": 1e-05,
22
+ "num_heads": 4,
23
+ "num_hidden_layers": 35,
24
+ "q_lora_rank": 32,
25
+ "qk_head_dim": 96,
26
+ "qk_nope_head_dim": 64,
27
+ "qk_rope_head_dim": 32,
28
+ "rope_scaling": null,
29
+ "rope_theta": 10000.0,
30
+ "tie_word_embeddings": true,
31
+ "transformers_version": "4.57.1",
32
+ "use_cache": true,
33
+ "use_l2warp": false,
34
+ "v_head_dim": 32,
35
+ "vocab_size": 256,
36
+ "window_size": null
37
+ }
checkpoint-5209/generation_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 0,
4
+ "eos_token_id": 1,
5
+ "transformers_version": "4.57.1"
6
+ }
checkpoint-5209/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c0b0e02995fa075d7e0662517e14fd69461add8b38dafd18e07bb4d4037fbc98
3
+ size 111070184
checkpoint-5209/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9c63c530e8aec7ba5d729b269ea07d57ac6cf1142fc8224eef40be7367e2a587
3
+ size 222434622
checkpoint-5209/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4
3
+ size 14645
checkpoint-5209/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:65a44a6e2e60a8f39fb71000c42309300e3693ac736943861802b43e61162e87
3
+ size 1465
checkpoint-5209/special_tokens_map.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<think>",
4
+ "</think>"
5
+ ],
6
+ "bos_token": {
7
+ "content": "<|startoftext|>",
8
+ "lstrip": false,
9
+ "normalized": false,
10
+ "rstrip": false,
11
+ "single_word": false
12
+ },
13
+ "eos_token": {
14
+ "content": "<|endoftext|>",
15
+ "lstrip": false,
16
+ "normalized": false,
17
+ "rstrip": false,
18
+ "single_word": false
19
+ },
20
+ "pad_token": "<|pad|>",
21
+ "unk_token": {
22
+ "content": "<|unknown|>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false
27
+ }
28
+ }
checkpoint-5209/tokenizer.json ADDED
@@ -0,0 +1,1085 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "version": "1.0",
3
+ "truncation": null,
4
+ "padding": null,
5
+ "added_tokens": [
6
+ {
7
+ "id": 0,
8
+ "content": "<|startoftext|>",
9
+ "single_word": false,
10
+ "lstrip": false,
11
+ "rstrip": false,
12
+ "normalized": false,
13
+ "special": true
14
+ },
15
+ {
16
+ "id": 1,
17
+ "content": "<|endoftext|>",
18
+ "single_word": false,
19
+ "lstrip": false,
20
+ "rstrip": false,
21
+ "normalized": false,
22
+ "special": true
23
+ },
24
+ {
25
+ "id": 2,
26
+ "content": "<|unknown|>",
27
+ "single_word": false,
28
+ "lstrip": false,
29
+ "rstrip": false,
30
+ "normalized": false,
31
+ "special": true
32
+ },
33
+ {
34
+ "id": 3,
35
+ "content": "<|pad|>",
36
+ "single_word": false,
37
+ "lstrip": false,
38
+ "rstrip": false,
39
+ "normalized": false,
40
+ "special": true
41
+ },
42
+ {
43
+ "id": 4,
44
+ "content": "<think>",
45
+ "single_word": false,
46
+ "lstrip": false,
47
+ "rstrip": false,
48
+ "normalized": false,
49
+ "special": true
50
+ },
51
+ {
52
+ "id": 5,
53
+ "content": "</think>",
54
+ "single_word": false,
55
+ "lstrip": false,
56
+ "rstrip": false,
57
+ "normalized": false,
58
+ "special": true
59
+ }
60
+ ],
61
+ "normalizer": null,
62
+ "pre_tokenizer": {
63
+ "type": "ByteLevel",
64
+ "add_prefix_space": false,
65
+ "trim_offsets": true,
66
+ "use_regex": true
67
+ },
68
+ "post_processor": null,
69
+ "decoder": {
70
+ "type": "ByteLevel",
71
+ "add_prefix_space": true,
72
+ "trim_offsets": true,
73
+ "use_regex": true
74
+ },
75
+ "model": {
76
+ "type": "BPE",
77
+ "dropout": null,
78
+ "unk_token": null,
79
+ "continuing_subword_prefix": null,
80
+ "end_of_word_suffix": null,
81
+ "fuse_unk": false,
82
+ "byte_fallback": false,
83
+ "ignore_merges": false,
84
+ "vocab": {
85
+ "<|startoftext|>": 0,
86
+ "<|endoftext|>": 1,
87
+ "<|unknown|>": 2,
88
+ "<|pad|>": 3,
89
+ "<think>": 4,
90
+ "</think>": 5,
91
+ "%": 6,
92
+ "&": 7,
93
+ "'": 8,
94
+ "(": 9,
95
+ ")": 10,
96
+ ",": 11,
97
+ "-": 12,
98
+ ".": 13,
99
+ "0": 14,
100
+ "1": 15,
101
+ "2": 16,
102
+ "3": 17,
103
+ "4": 18,
104
+ "5": 19,
105
+ "6": 20,
106
+ "7": 21,
107
+ "8": 22,
108
+ "9": 23,
109
+ ":": 24,
110
+ ">": 25,
111
+ "A": 26,
112
+ "C": 27,
113
+ "D": 28,
114
+ "E": 29,
115
+ "F": 30,
116
+ "G": 31,
117
+ "H": 32,
118
+ "I": 33,
119
+ "L": 34,
120
+ "M": 35,
121
+ "N": 36,
122
+ "O": 37,
123
+ "P": 38,
124
+ "R": 39,
125
+ "S": 40,
126
+ "T": 41,
127
+ "V": 42,
128
+ "[": 43,
129
+ "]": 44,
130
+ "a": 45,
131
+ "b": 46,
132
+ "c": 47,
133
+ "d": 48,
134
+ "e": 49,
135
+ "f": 50,
136
+ "g": 51,
137
+ "h": 52,
138
+ "i": 53,
139
+ "k": 54,
140
+ "l": 55,
141
+ "m": 56,
142
+ "n": 57,
143
+ "o": 58,
144
+ "p": 59,
145
+ "q": 60,
146
+ "r": 61,
147
+ "s": 62,
148
+ "t": 63,
149
+ "u": 64,
150
+ "v": 65,
151
+ "x": 66,
152
+ "y": 67,
153
+ "|": 68,
154
+ "Ċ": 69,
155
+ "Ġ": 70,
156
+ "00": 71,
157
+ "11": 72,
158
+ "->": 73,
159
+ "Ġ|": 74,
160
+ ":[": 75,
161
+ "]->": 76,
162
+ "10": 77,
163
+ "Ġ1": 78,
164
+ "000": 79,
165
+ "ĠS": 80,
166
+ "',": 81,
167
+ "Ġ'": 82,
168
+ "te": 83,
169
+ "100": 84,
170
+ "ĊĠ": 85,
171
+ "110": 86,
172
+ "tep": 87,
173
+ "re": 88,
174
+ "111": 89,
175
+ "0000": 90,
176
+ "011": 91,
177
+ "101": 92,
178
+ "001": 93,
179
+ "ĠStep": 94,
180
+ "010": 95,
181
+ "in": 96,
182
+ "im": 97,
183
+ "ĠSim": 98,
184
+ "1111": 99,
185
+ "Ġ[": 100,
186
+ "or": 101,
187
+ "ĠI": 102,
188
+ "ho": 103,
189
+ "od": 104,
190
+ "hood": 105,
191
+ "Ġin": 106,
192
+ "Ġ11": 107,
193
+ "Ġ10": 108,
194
+ "ne": 109,
195
+ "Ġne": 110,
196
+ "']": 111,
197
+ "bor": 112,
198
+ "dx": 113,
199
+ "gh": 114,
200
+ "ire": 115,
201
+ "igh": 116,
202
+ "qu": 117,
203
+ "Ġre": 118,
204
+ "Ġ['": 119,
205
+ "ĠIdx": 120,
206
+ "Ġneigh": 121,
207
+ "borhood": 122,
208
+ "ires": 123,
209
+ "quires": 124,
210
+ "Ġrequires": 125,
211
+ "Ġneighborhood": 126,
212
+ "Step": 127,
213
+ "ut": 128,
214
+ "Ġ3": 129,
215
+ "Ġ2": 130,
216
+ "Ġ4": 131,
217
+ "Ġ5": 132,
218
+ "Ġ6": 133,
219
+ "Ġ9": 134,
220
+ "Ġ8": 135,
221
+ "Ġ7": 136,
222
+ "Ġ0": 137,
223
+ "ca": 138,
224
+ "Ġ12": 139,
225
+ "Ġ13": 140,
226
+ "1010": 141,
227
+ "Ġ14": 142,
228
+ "Out": 143,
229
+ "put": 144,
230
+ "ĠOut": 145,
231
+ "ĠSca": 146,
232
+ "ĠOutput": 147,
233
+ "ĠScan": 148,
234
+ "Ġ011": 149,
235
+ "ing": 150,
236
+ "ti": 151,
237
+ "1100": 152,
238
+ "00000000": 153,
239
+ "Ġ15": 154,
240
+ "Ġ000": 155,
241
+ "on": 156,
242
+ "Ġ110": 157,
243
+ "ul": 158,
244
+ "Ġ100": 159,
245
+ "at": 160,
246
+ "tion": 161,
247
+ "Ġ010": 162,
248
+ "1011": 163,
249
+ "ĠC": 164,
250
+ "Ġ00": 165,
251
+ "00011": 166,
252
+ "50": 167,
253
+ "er": 168,
254
+ "10000": 169,
255
+ "nd": 170,
256
+ "Ġ16": 171,
257
+ "ule": 172,
258
+ "ĠR": 173,
259
+ "ed": 174,
260
+ "li": 175,
261
+ "Ġ001": 176,
262
+ "Ġ101": 177,
263
+ "ate": 178,
264
+ "ĠRule": 179,
265
+ "1000": 180,
266
+ "fi": 181,
267
+ "10100": 182,
268
+ "as": 183,
269
+ "Ġ17": 184,
270
+ "ĠM": 185,
271
+ "Ġ1111": 186,
272
+ "Ver": 187,
273
+ "ch": 188,
274
+ "es": 189,
275
+ "ifi": 190,
276
+ "mp": 191,
277
+ "ting": 192,
278
+ "cation": 193,
279
+ "atch": 194,
280
+ "Verifi": 195,
281
+ "Verification": 196,
282
+ "..": 197,
283
+ "11011": 198,
284
+ "41": 199,
285
+ "and": 200,
286
+ "date": 201,
287
+ "idate": 202,
288
+ "ĠCand": 203,
289
+ "ĠCandidate": 204,
290
+ "58": 205,
291
+ "ating": 206,
292
+ "10011": 207,
293
+ "ction": 208,
294
+ "111000": 209,
295
+ "000000": 210,
296
+ "10110": 211,
297
+ "Ġ00000000": 212,
298
+ "is": 213,
299
+ "match": 214,
300
+ "ĠMis": 215,
301
+ "ĠMismatch": 216,
302
+ "111111": 217,
303
+ "Tas": 218,
304
+ "Task": 219,
305
+ "Ġ18": 220,
306
+ "Gre": 221,
307
+ "Pr": 222,
308
+ "ĠL": 223,
309
+ "ĠGre": 224,
310
+ "ĠPr": 225,
311
+ "edy": 226,
312
+ "ĠGreedy": 227,
313
+ "At": 228,
314
+ "Ex": 229,
315
+ "Tes": 230,
316
+ "ĠTes": 231,
317
+ "temp": 232,
318
+ "Attemp": 233,
319
+ "ĠTesting": 234,
320
+ "Attempt": 235,
321
+ "100000": 236,
322
+ "Ġs": 237,
323
+ "Ġ0000": 238,
324
+ "uction": 239,
325
+ "ĠSteps": 240,
326
+ "xt": 241,
327
+ "101111": 242,
328
+ "al": 243,
329
+ "cul": 244,
330
+ "eli": 245,
331
+ "ior": 246,
332
+ "keli": 247,
333
+ "ver": 248,
334
+ "cal": 249,
335
+ "culating": 250,
336
+ "iors": 251,
337
+ "kelihood": 252,
338
+ "Ġ19": 253,
339
+ "11111111": 254,
340
+ ").": 255
341
+ },
342
+ "merges": [
343
+ [
344
+ "0",
345
+ "0"
346
+ ],
347
+ [
348
+ "1",
349
+ "1"
350
+ ],
351
+ [
352
+ "-",
353
+ ">"
354
+ ],
355
+ [
356
+ "Ġ",
357
+ "|"
358
+ ],
359
+ [
360
+ ":",
361
+ "["
362
+ ],
363
+ [
364
+ "]",
365
+ "->"
366
+ ],
367
+ [
368
+ "1",
369
+ "0"
370
+ ],
371
+ [
372
+ "Ġ",
373
+ "1"
374
+ ],
375
+ [
376
+ "00",
377
+ "0"
378
+ ],
379
+ [
380
+ "Ġ",
381
+ "S"
382
+ ],
383
+ [
384
+ "'",
385
+ ","
386
+ ],
387
+ [
388
+ "Ġ",
389
+ "'"
390
+ ],
391
+ [
392
+ "t",
393
+ "e"
394
+ ],
395
+ [
396
+ "1",
397
+ "00"
398
+ ],
399
+ [
400
+ "Ċ",
401
+ "Ġ"
402
+ ],
403
+ [
404
+ "11",
405
+ "0"
406
+ ],
407
+ [
408
+ "te",
409
+ "p"
410
+ ],
411
+ [
412
+ "r",
413
+ "e"
414
+ ],
415
+ [
416
+ "11",
417
+ "1"
418
+ ],
419
+ [
420
+ "00",
421
+ "00"
422
+ ],
423
+ [
424
+ "0",
425
+ "11"
426
+ ],
427
+ [
428
+ "10",
429
+ "1"
430
+ ],
431
+ [
432
+ "00",
433
+ "1"
434
+ ],
435
+ [
436
+ "ĠS",
437
+ "tep"
438
+ ],
439
+ [
440
+ "0",
441
+ "10"
442
+ ],
443
+ [
444
+ "i",
445
+ "n"
446
+ ],
447
+ [
448
+ "i",
449
+ "m"
450
+ ],
451
+ [
452
+ "ĠS",
453
+ "im"
454
+ ],
455
+ [
456
+ "11",
457
+ "11"
458
+ ],
459
+ [
460
+ "Ġ",
461
+ "["
462
+ ],
463
+ [
464
+ "o",
465
+ "r"
466
+ ],
467
+ [
468
+ "Ġ",
469
+ "I"
470
+ ],
471
+ [
472
+ "h",
473
+ "o"
474
+ ],
475
+ [
476
+ "o",
477
+ "d"
478
+ ],
479
+ [
480
+ "ho",
481
+ "od"
482
+ ],
483
+ [
484
+ "Ġ",
485
+ "in"
486
+ ],
487
+ [
488
+ "Ġ",
489
+ "11"
490
+ ],
491
+ [
492
+ "Ġ",
493
+ "10"
494
+ ],
495
+ [
496
+ "n",
497
+ "e"
498
+ ],
499
+ [
500
+ "Ġ",
501
+ "ne"
502
+ ],
503
+ [
504
+ "'",
505
+ "]"
506
+ ],
507
+ [
508
+ "b",
509
+ "or"
510
+ ],
511
+ [
512
+ "d",
513
+ "x"
514
+ ],
515
+ [
516
+ "g",
517
+ "h"
518
+ ],
519
+ [
520
+ "i",
521
+ "re"
522
+ ],
523
+ [
524
+ "i",
525
+ "gh"
526
+ ],
527
+ [
528
+ "q",
529
+ "u"
530
+ ],
531
+ [
532
+ "Ġ",
533
+ "re"
534
+ ],
535
+ [
536
+ "Ġ[",
537
+ "'"
538
+ ],
539
+ [
540
+ "ĠI",
541
+ "dx"
542
+ ],
543
+ [
544
+ "Ġne",
545
+ "igh"
546
+ ],
547
+ [
548
+ "bor",
549
+ "hood"
550
+ ],
551
+ [
552
+ "ire",
553
+ "s"
554
+ ],
555
+ [
556
+ "qu",
557
+ "ires"
558
+ ],
559
+ [
560
+ "Ġre",
561
+ "quires"
562
+ ],
563
+ [
564
+ "Ġneigh",
565
+ "borhood"
566
+ ],
567
+ [
568
+ "S",
569
+ "tep"
570
+ ],
571
+ [
572
+ "u",
573
+ "t"
574
+ ],
575
+ [
576
+ "Ġ",
577
+ "3"
578
+ ],
579
+ [
580
+ "Ġ",
581
+ "2"
582
+ ],
583
+ [
584
+ "Ġ",
585
+ "4"
586
+ ],
587
+ [
588
+ "Ġ",
589
+ "5"
590
+ ],
591
+ [
592
+ "Ġ",
593
+ "6"
594
+ ],
595
+ [
596
+ "Ġ",
597
+ "9"
598
+ ],
599
+ [
600
+ "Ġ",
601
+ "8"
602
+ ],
603
+ [
604
+ "Ġ",
605
+ "7"
606
+ ],
607
+ [
608
+ "Ġ",
609
+ "0"
610
+ ],
611
+ [
612
+ "c",
613
+ "a"
614
+ ],
615
+ [
616
+ "Ġ1",
617
+ "2"
618
+ ],
619
+ [
620
+ "Ġ1",
621
+ "3"
622
+ ],
623
+ [
624
+ "10",
625
+ "10"
626
+ ],
627
+ [
628
+ "Ġ1",
629
+ "4"
630
+ ],
631
+ [
632
+ "O",
633
+ "ut"
634
+ ],
635
+ [
636
+ "p",
637
+ "ut"
638
+ ],
639
+ [
640
+ "Ġ",
641
+ "Out"
642
+ ],
643
+ [
644
+ "ĠS",
645
+ "ca"
646
+ ],
647
+ [
648
+ "ĠOut",
649
+ "put"
650
+ ],
651
+ [
652
+ "ĠSca",
653
+ "n"
654
+ ],
655
+ [
656
+ "Ġ",
657
+ "011"
658
+ ],
659
+ [
660
+ "in",
661
+ "g"
662
+ ],
663
+ [
664
+ "t",
665
+ "i"
666
+ ],
667
+ [
668
+ "11",
669
+ "00"
670
+ ],
671
+ [
672
+ "0000",
673
+ "0000"
674
+ ],
675
+ [
676
+ "Ġ1",
677
+ "5"
678
+ ],
679
+ [
680
+ "Ġ",
681
+ "000"
682
+ ],
683
+ [
684
+ "o",
685
+ "n"
686
+ ],
687
+ [
688
+ "Ġ",
689
+ "110"
690
+ ],
691
+ [
692
+ "u",
693
+ "l"
694
+ ],
695
+ [
696
+ "Ġ1",
697
+ "00"
698
+ ],
699
+ [
700
+ "a",
701
+ "t"
702
+ ],
703
+ [
704
+ "ti",
705
+ "on"
706
+ ],
707
+ [
708
+ "Ġ",
709
+ "010"
710
+ ],
711
+ [
712
+ "10",
713
+ "11"
714
+ ],
715
+ [
716
+ "Ġ",
717
+ "C"
718
+ ],
719
+ [
720
+ "Ġ",
721
+ "00"
722
+ ],
723
+ [
724
+ "000",
725
+ "11"
726
+ ],
727
+ [
728
+ "5",
729
+ "0"
730
+ ],
731
+ [
732
+ "e",
733
+ "r"
734
+ ],
735
+ [
736
+ "100",
737
+ "00"
738
+ ],
739
+ [
740
+ "n",
741
+ "d"
742
+ ],
743
+ [
744
+ "Ġ1",
745
+ "6"
746
+ ],
747
+ [
748
+ "ul",
749
+ "e"
750
+ ],
751
+ [
752
+ "Ġ",
753
+ "R"
754
+ ],
755
+ [
756
+ "e",
757
+ "d"
758
+ ],
759
+ [
760
+ "l",
761
+ "i"
762
+ ],
763
+ [
764
+ "Ġ",
765
+ "001"
766
+ ],
767
+ [
768
+ "Ġ",
769
+ "101"
770
+ ],
771
+ [
772
+ "a",
773
+ "te"
774
+ ],
775
+ [
776
+ "ĠR",
777
+ "ule"
778
+ ],
779
+ [
780
+ "1",
781
+ "000"
782
+ ],
783
+ [
784
+ "f",
785
+ "i"
786
+ ],
787
+ [
788
+ "10",
789
+ "100"
790
+ ],
791
+ [
792
+ "a",
793
+ "s"
794
+ ],
795
+ [
796
+ "Ġ1",
797
+ "7"
798
+ ],
799
+ [
800
+ "Ġ",
801
+ "M"
802
+ ],
803
+ [
804
+ "Ġ",
805
+ "1111"
806
+ ],
807
+ [
808
+ "V",
809
+ "er"
810
+ ],
811
+ [
812
+ "c",
813
+ "h"
814
+ ],
815
+ [
816
+ "e",
817
+ "s"
818
+ ],
819
+ [
820
+ "i",
821
+ "fi"
822
+ ],
823
+ [
824
+ "m",
825
+ "p"
826
+ ],
827
+ [
828
+ "t",
829
+ "ing"
830
+ ],
831
+ [
832
+ "ca",
833
+ "tion"
834
+ ],
835
+ [
836
+ "at",
837
+ "ch"
838
+ ],
839
+ [
840
+ "Ver",
841
+ "ifi"
842
+ ],
843
+ [
844
+ "Verifi",
845
+ "cation"
846
+ ],
847
+ [
848
+ ".",
849
+ "."
850
+ ],
851
+ [
852
+ "110",
853
+ "11"
854
+ ],
855
+ [
856
+ "4",
857
+ "1"
858
+ ],
859
+ [
860
+ "a",
861
+ "nd"
862
+ ],
863
+ [
864
+ "d",
865
+ "ate"
866
+ ],
867
+ [
868
+ "i",
869
+ "date"
870
+ ],
871
+ [
872
+ "ĠC",
873
+ "and"
874
+ ],
875
+ [
876
+ "ĠCand",
877
+ "idate"
878
+ ],
879
+ [
880
+ "5",
881
+ "8"
882
+ ],
883
+ [
884
+ "at",
885
+ "ing"
886
+ ],
887
+ [
888
+ "100",
889
+ "11"
890
+ ],
891
+ [
892
+ "c",
893
+ "tion"
894
+ ],
895
+ [
896
+ "111",
897
+ "000"
898
+ ],
899
+ [
900
+ "0000",
901
+ "00"
902
+ ],
903
+ [
904
+ "10",
905
+ "110"
906
+ ],
907
+ [
908
+ "Ġ",
909
+ "00000000"
910
+ ],
911
+ [
912
+ "i",
913
+ "s"
914
+ ],
915
+ [
916
+ "m",
917
+ "atch"
918
+ ],
919
+ [
920
+ "ĠM",
921
+ "is"
922
+ ],
923
+ [
924
+ "ĠMis",
925
+ "match"
926
+ ],
927
+ [
928
+ "1111",
929
+ "11"
930
+ ],
931
+ [
932
+ "T",
933
+ "as"
934
+ ],
935
+ [
936
+ "Tas",
937
+ "k"
938
+ ],
939
+ [
940
+ "Ġ1",
941
+ "8"
942
+ ],
943
+ [
944
+ "G",
945
+ "re"
946
+ ],
947
+ [
948
+ "P",
949
+ "r"
950
+ ],
951
+ [
952
+ "Ġ",
953
+ "L"
954
+ ],
955
+ [
956
+ "Ġ",
957
+ "Gre"
958
+ ],
959
+ [
960
+ "Ġ",
961
+ "Pr"
962
+ ],
963
+ [
964
+ "ed",
965
+ "y"
966
+ ],
967
+ [
968
+ "ĠGre",
969
+ "edy"
970
+ ],
971
+ [
972
+ "A",
973
+ "t"
974
+ ],
975
+ [
976
+ "E",
977
+ "x"
978
+ ],
979
+ [
980
+ "T",
981
+ "es"
982
+ ],
983
+ [
984
+ "Ġ",
985
+ "Tes"
986
+ ],
987
+ [
988
+ "te",
989
+ "mp"
990
+ ],
991
+ [
992
+ "At",
993
+ "temp"
994
+ ],
995
+ [
996
+ "ĠTes",
997
+ "ting"
998
+ ],
999
+ [
1000
+ "Attemp",
1001
+ "t"
1002
+ ],
1003
+ [
1004
+ "100",
1005
+ "000"
1006
+ ],
1007
+ [
1008
+ "Ġ",
1009
+ "s"
1010
+ ],
1011
+ [
1012
+ "Ġ",
1013
+ "0000"
1014
+ ],
1015
+ [
1016
+ "u",
1017
+ "ction"
1018
+ ],
1019
+ [
1020
+ "ĠStep",
1021
+ "s"
1022
+ ],
1023
+ [
1024
+ "x",
1025
+ "t"
1026
+ ],
1027
+ [
1028
+ "10",
1029
+ "1111"
1030
+ ],
1031
+ [
1032
+ "a",
1033
+ "l"
1034
+ ],
1035
+ [
1036
+ "c",
1037
+ "ul"
1038
+ ],
1039
+ [
1040
+ "e",
1041
+ "li"
1042
+ ],
1043
+ [
1044
+ "i",
1045
+ "or"
1046
+ ],
1047
+ [
1048
+ "k",
1049
+ "eli"
1050
+ ],
1051
+ [
1052
+ "v",
1053
+ "er"
1054
+ ],
1055
+ [
1056
+ "ca",
1057
+ "l"
1058
+ ],
1059
+ [
1060
+ "cul",
1061
+ "ating"
1062
+ ],
1063
+ [
1064
+ "ior",
1065
+ "s"
1066
+ ],
1067
+ [
1068
+ "keli",
1069
+ "hood"
1070
+ ],
1071
+ [
1072
+ "Ġ1",
1073
+ "9"
1074
+ ],
1075
+ [
1076
+ "1111",
1077
+ "1111"
1078
+ ],
1079
+ [
1080
+ ")",
1081
+ "."
1082
+ ]
1083
+ ]
1084
+ }
1085
+ }
checkpoint-5209/tokenizer_config.json ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<|startoftext|>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "<|endoftext|>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "<|unknown|>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "3": {
28
+ "content": "<|pad|>",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "4": {
36
+ "content": "<think>",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ },
43
+ "5": {
44
+ "content": "</think>",
45
+ "lstrip": false,
46
+ "normalized": false,
47
+ "rstrip": false,
48
+ "single_word": false,
49
+ "special": true
50
+ }
51
+ },
52
+ "additional_special_tokens": [
53
+ "<think>",
54
+ "</think>"
55
+ ],
56
+ "bos_token": "<|startoftext|>",
57
+ "clean_up_tokenization_spaces": false,
58
+ "eos_token": "<|endoftext|>",
59
+ "extra_special_tokens": {},
60
+ "model_max_length": 1000000000000000019884624838656,
61
+ "pad_token": "<|pad|>",
62
+ "tokenizer_class": "PreTrainedTokenizerFast",
63
+ "unk_token": "<|unknown|>"
64
+ }
checkpoint-5209/trainer_state.json ADDED
@@ -0,0 +1,762 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 1.0,
6
+ "eval_steps": 500,
7
+ "global_step": 5209,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.009599692809830085,
14
+ "grad_norm": 14.53178596496582,
15
+ "learning_rate": 1.47e-05,
16
+ "loss": 18.117,
17
+ "step": 50
18
+ },
19
+ {
20
+ "epoch": 0.01919938561966017,
21
+ "grad_norm": 8.379057884216309,
22
+ "learning_rate": 2.97e-05,
23
+ "loss": 11.1278,
24
+ "step": 100
25
+ },
26
+ {
27
+ "epoch": 0.028799078429490255,
28
+ "grad_norm": 17.656112670898438,
29
+ "learning_rate": 4.4699999999999996e-05,
30
+ "loss": 8.8796,
31
+ "step": 150
32
+ },
33
+ {
34
+ "epoch": 0.03839877123932034,
35
+ "grad_norm": 14.292523384094238,
36
+ "learning_rate": 5.97e-05,
37
+ "loss": 7.8564,
38
+ "step": 200
39
+ },
40
+ {
41
+ "epoch": 0.047998464049150424,
42
+ "grad_norm": 19.745756149291992,
43
+ "learning_rate": 7.47e-05,
44
+ "loss": 7.1001,
45
+ "step": 250
46
+ },
47
+ {
48
+ "epoch": 0.05759815685898051,
49
+ "grad_norm": 45.06932830810547,
50
+ "learning_rate": 8.969999999999998e-05,
51
+ "loss": 6.4644,
52
+ "step": 300
53
+ },
54
+ {
55
+ "epoch": 0.0671978496688106,
56
+ "grad_norm": 26.414615631103516,
57
+ "learning_rate": 0.00010469999999999998,
58
+ "loss": 5.9291,
59
+ "step": 350
60
+ },
61
+ {
62
+ "epoch": 0.07679754247864068,
63
+ "grad_norm": 21.97804069519043,
64
+ "learning_rate": 0.0001197,
65
+ "loss": 5.367,
66
+ "step": 400
67
+ },
68
+ {
69
+ "epoch": 0.08639723528847076,
70
+ "grad_norm": 21.083677291870117,
71
+ "learning_rate": 0.0001347,
72
+ "loss": 4.7741,
73
+ "step": 450
74
+ },
75
+ {
76
+ "epoch": 0.09599692809830085,
77
+ "grad_norm": 34.146934509277344,
78
+ "learning_rate": 0.00014969999999999998,
79
+ "loss": 4.3934,
80
+ "step": 500
81
+ },
82
+ {
83
+ "epoch": 0.10559662090813093,
84
+ "grad_norm": 31.148387908935547,
85
+ "learning_rate": 0.0001647,
86
+ "loss": 4.008,
87
+ "step": 550
88
+ },
89
+ {
90
+ "epoch": 0.11519631371796102,
91
+ "grad_norm": 21.12940788269043,
92
+ "learning_rate": 0.00017969999999999998,
93
+ "loss": 3.531,
94
+ "step": 600
95
+ },
96
+ {
97
+ "epoch": 0.12479600652779112,
98
+ "grad_norm": 13.038416862487793,
99
+ "learning_rate": 0.0001947,
100
+ "loss": 2.9393,
101
+ "step": 650
102
+ },
103
+ {
104
+ "epoch": 0.1343956993376212,
105
+ "grad_norm": 6.97022819519043,
106
+ "learning_rate": 0.00020969999999999997,
107
+ "loss": 2.4658,
108
+ "step": 700
109
+ },
110
+ {
111
+ "epoch": 0.14399539214745127,
112
+ "grad_norm": 11.445544242858887,
113
+ "learning_rate": 0.0002247,
114
+ "loss": 2.1561,
115
+ "step": 750
116
+ },
117
+ {
118
+ "epoch": 0.15359508495728136,
119
+ "grad_norm": 9.814635276794434,
120
+ "learning_rate": 0.0002397,
121
+ "loss": 1.8553,
122
+ "step": 800
123
+ },
124
+ {
125
+ "epoch": 0.16319477776711144,
126
+ "grad_norm": 10.05532169342041,
127
+ "learning_rate": 0.00025469999999999996,
128
+ "loss": 1.6969,
129
+ "step": 850
130
+ },
131
+ {
132
+ "epoch": 0.17279447057694153,
133
+ "grad_norm": 6.568735122680664,
134
+ "learning_rate": 0.0002697,
135
+ "loss": 1.5864,
136
+ "step": 900
137
+ },
138
+ {
139
+ "epoch": 0.1823941633867716,
140
+ "grad_norm": 11.432738304138184,
141
+ "learning_rate": 0.0002847,
142
+ "loss": 1.4549,
143
+ "step": 950
144
+ },
145
+ {
146
+ "epoch": 0.1919938561966017,
147
+ "grad_norm": 7.344091415405273,
148
+ "learning_rate": 0.00029969999999999997,
149
+ "loss": 1.3058,
150
+ "step": 1000
151
+ },
152
+ {
153
+ "epoch": 0.20159354900643178,
154
+ "grad_norm": 7.833723068237305,
155
+ "learning_rate": 0.0002965074839629365,
156
+ "loss": 1.2029,
157
+ "step": 1050
158
+ },
159
+ {
160
+ "epoch": 0.21119324181626187,
161
+ "grad_norm": 9.809481620788574,
162
+ "learning_rate": 0.000292943692088382,
163
+ "loss": 1.1299,
164
+ "step": 1100
165
+ },
166
+ {
167
+ "epoch": 0.22079293462609195,
168
+ "grad_norm": 10.855881690979004,
169
+ "learning_rate": 0.0002893799002138275,
170
+ "loss": 1.0214,
171
+ "step": 1150
172
+ },
173
+ {
174
+ "epoch": 0.23039262743592204,
175
+ "grad_norm": 2.817777156829834,
176
+ "learning_rate": 0.000285816108339273,
177
+ "loss": 0.9556,
178
+ "step": 1200
179
+ },
180
+ {
181
+ "epoch": 0.23999232024575212,
182
+ "grad_norm": 5.34994649887085,
183
+ "learning_rate": 0.00028225231646471845,
184
+ "loss": 0.8728,
185
+ "step": 1250
186
+ },
187
+ {
188
+ "epoch": 0.24959201305558223,
189
+ "grad_norm": 1.8884717226028442,
190
+ "learning_rate": 0.0002786885245901639,
191
+ "loss": 0.7461,
192
+ "step": 1300
193
+ },
194
+ {
195
+ "epoch": 0.2591917058654123,
196
+ "grad_norm": 1.8414419889450073,
197
+ "learning_rate": 0.0002751247327156094,
198
+ "loss": 0.6367,
199
+ "step": 1350
200
+ },
201
+ {
202
+ "epoch": 0.2687913986752424,
203
+ "grad_norm": 1.6845418214797974,
204
+ "learning_rate": 0.00027156094084105487,
205
+ "loss": 0.5393,
206
+ "step": 1400
207
+ },
208
+ {
209
+ "epoch": 0.27839109148507246,
210
+ "grad_norm": 0.8279030323028564,
211
+ "learning_rate": 0.00026799714896650034,
212
+ "loss": 0.4878,
213
+ "step": 1450
214
+ },
215
+ {
216
+ "epoch": 0.28799078429490255,
217
+ "grad_norm": 1.0994820594787598,
218
+ "learning_rate": 0.0002644333570919458,
219
+ "loss": 0.4248,
220
+ "step": 1500
221
+ },
222
+ {
223
+ "epoch": 0.29759047710473263,
224
+ "grad_norm": 1.264716625213623,
225
+ "learning_rate": 0.0002608695652173913,
226
+ "loss": 0.3817,
227
+ "step": 1550
228
+ },
229
+ {
230
+ "epoch": 0.3071901699145627,
231
+ "grad_norm": 0.8758682012557983,
232
+ "learning_rate": 0.00025730577334283675,
233
+ "loss": 0.3623,
234
+ "step": 1600
235
+ },
236
+ {
237
+ "epoch": 0.3167898627243928,
238
+ "grad_norm": 0.5896995663642883,
239
+ "learning_rate": 0.0002537419814682822,
240
+ "loss": 0.3208,
241
+ "step": 1650
242
+ },
243
+ {
244
+ "epoch": 0.3263895555342229,
245
+ "grad_norm": 0.7703513503074646,
246
+ "learning_rate": 0.0002501781895937277,
247
+ "loss": 0.2889,
248
+ "step": 1700
249
+ },
250
+ {
251
+ "epoch": 0.33598924834405297,
252
+ "grad_norm": 0.5982457399368286,
253
+ "learning_rate": 0.00024661439771917317,
254
+ "loss": 0.2803,
255
+ "step": 1750
256
+ },
257
+ {
258
+ "epoch": 0.34558894115388306,
259
+ "grad_norm": 0.5336303114891052,
260
+ "learning_rate": 0.00024305060584461867,
261
+ "loss": 0.3014,
262
+ "step": 1800
263
+ },
264
+ {
265
+ "epoch": 0.35518863396371314,
266
+ "grad_norm": 1.1344468593597412,
267
+ "learning_rate": 0.00023948681397006414,
268
+ "loss": 0.2615,
269
+ "step": 1850
270
+ },
271
+ {
272
+ "epoch": 0.3647883267735432,
273
+ "grad_norm": 1.2401304244995117,
274
+ "learning_rate": 0.0002359230220955096,
275
+ "loss": 0.2436,
276
+ "step": 1900
277
+ },
278
+ {
279
+ "epoch": 0.3743880195833733,
280
+ "grad_norm": 0.8760889172554016,
281
+ "learning_rate": 0.00023235923022095505,
282
+ "loss": 0.221,
283
+ "step": 1950
284
+ },
285
+ {
286
+ "epoch": 0.3839877123932034,
287
+ "grad_norm": 0.6058375239372253,
288
+ "learning_rate": 0.00022879543834640053,
289
+ "loss": 0.2061,
290
+ "step": 2000
291
+ },
292
+ {
293
+ "epoch": 0.3935874052030335,
294
+ "grad_norm": 0.5243856906890869,
295
+ "learning_rate": 0.00022523164647184602,
296
+ "loss": 0.1934,
297
+ "step": 2050
298
+ },
299
+ {
300
+ "epoch": 0.40318709801286357,
301
+ "grad_norm": 0.45033150911331177,
302
+ "learning_rate": 0.0002216678545972915,
303
+ "loss": 0.1869,
304
+ "step": 2100
305
+ },
306
+ {
307
+ "epoch": 0.41278679082269365,
308
+ "grad_norm": 0.5008948445320129,
309
+ "learning_rate": 0.00021810406272273697,
310
+ "loss": 0.1843,
311
+ "step": 2150
312
+ },
313
+ {
314
+ "epoch": 0.42238648363252373,
315
+ "grad_norm": 1.0652313232421875,
316
+ "learning_rate": 0.00021454027084818244,
317
+ "loss": 0.1811,
318
+ "step": 2200
319
+ },
320
+ {
321
+ "epoch": 0.4319861764423538,
322
+ "grad_norm": 0.3990040421485901,
323
+ "learning_rate": 0.00021097647897362794,
324
+ "loss": 0.1791,
325
+ "step": 2250
326
+ },
327
+ {
328
+ "epoch": 0.4415858692521839,
329
+ "grad_norm": 0.4828120172023773,
330
+ "learning_rate": 0.0002074126870990734,
331
+ "loss": 0.1681,
332
+ "step": 2300
333
+ },
334
+ {
335
+ "epoch": 0.451185562062014,
336
+ "grad_norm": 0.5252107977867126,
337
+ "learning_rate": 0.00020384889522451888,
338
+ "loss": 0.164,
339
+ "step": 2350
340
+ },
341
+ {
342
+ "epoch": 0.4607852548718441,
343
+ "grad_norm": 0.24024103581905365,
344
+ "learning_rate": 0.00020028510334996435,
345
+ "loss": 0.1518,
346
+ "step": 2400
347
+ },
348
+ {
349
+ "epoch": 0.47038494768167416,
350
+ "grad_norm": 0.5373029708862305,
351
+ "learning_rate": 0.0001967213114754098,
352
+ "loss": 0.1572,
353
+ "step": 2450
354
+ },
355
+ {
356
+ "epoch": 0.47998464049150424,
357
+ "grad_norm": 0.41635262966156006,
358
+ "learning_rate": 0.00019315751960085527,
359
+ "loss": 0.1459,
360
+ "step": 2500
361
+ },
362
+ {
363
+ "epoch": 0.4895843333013344,
364
+ "grad_norm": 0.26312512159347534,
365
+ "learning_rate": 0.00018959372772630077,
366
+ "loss": 0.1444,
367
+ "step": 2550
368
+ },
369
+ {
370
+ "epoch": 0.49918402611116447,
371
+ "grad_norm": 0.33238592743873596,
372
+ "learning_rate": 0.00018602993585174624,
373
+ "loss": 0.1415,
374
+ "step": 2600
375
+ },
376
+ {
377
+ "epoch": 0.5087837189209945,
378
+ "grad_norm": 0.4475176930427551,
379
+ "learning_rate": 0.0001824661439771917,
380
+ "loss": 0.1446,
381
+ "step": 2650
382
+ },
383
+ {
384
+ "epoch": 0.5183834117308246,
385
+ "grad_norm": 0.4537954032421112,
386
+ "learning_rate": 0.00017890235210263718,
387
+ "loss": 0.1395,
388
+ "step": 2700
389
+ },
390
+ {
391
+ "epoch": 0.5279831045406547,
392
+ "grad_norm": 0.3630366921424866,
393
+ "learning_rate": 0.00017533856022808268,
394
+ "loss": 0.1371,
395
+ "step": 2750
396
+ },
397
+ {
398
+ "epoch": 0.5375827973504848,
399
+ "grad_norm": 0.44030898809432983,
400
+ "learning_rate": 0.00017177476835352815,
401
+ "loss": 0.1377,
402
+ "step": 2800
403
+ },
404
+ {
405
+ "epoch": 0.5471824901603148,
406
+ "grad_norm": 0.4108527898788452,
407
+ "learning_rate": 0.00016821097647897363,
408
+ "loss": 0.1331,
409
+ "step": 2850
410
+ },
411
+ {
412
+ "epoch": 0.5567821829701449,
413
+ "grad_norm": 0.45702821016311646,
414
+ "learning_rate": 0.0001646471846044191,
415
+ "loss": 0.1325,
416
+ "step": 2900
417
+ },
418
+ {
419
+ "epoch": 0.566381875779975,
420
+ "grad_norm": 0.46175822615623474,
421
+ "learning_rate": 0.00016108339272986454,
422
+ "loss": 0.1273,
423
+ "step": 2950
424
+ },
425
+ {
426
+ "epoch": 0.5759815685898051,
427
+ "grad_norm": 0.41439440846443176,
428
+ "learning_rate": 0.00015751960085531,
429
+ "loss": 0.1309,
430
+ "step": 3000
431
+ },
432
+ {
433
+ "epoch": 0.5855812613996352,
434
+ "grad_norm": 0.3587839901447296,
435
+ "learning_rate": 0.0001539558089807555,
436
+ "loss": 0.1242,
437
+ "step": 3050
438
+ },
439
+ {
440
+ "epoch": 0.5951809542094653,
441
+ "grad_norm": 0.42960047721862793,
442
+ "learning_rate": 0.00015039201710620098,
443
+ "loss": 0.127,
444
+ "step": 3100
445
+ },
446
+ {
447
+ "epoch": 0.6047806470192953,
448
+ "grad_norm": 0.2696012854576111,
449
+ "learning_rate": 0.00014682822523164645,
450
+ "loss": 0.1253,
451
+ "step": 3150
452
+ },
453
+ {
454
+ "epoch": 0.6143803398291254,
455
+ "grad_norm": 0.2405436486005783,
456
+ "learning_rate": 0.00014326443335709193,
457
+ "loss": 0.121,
458
+ "step": 3200
459
+ },
460
+ {
461
+ "epoch": 0.6239800326389555,
462
+ "grad_norm": 0.20173802971839905,
463
+ "learning_rate": 0.00013970064148253743,
464
+ "loss": 0.1211,
465
+ "step": 3250
466
+ },
467
+ {
468
+ "epoch": 0.6335797254487856,
469
+ "grad_norm": 0.37551426887512207,
470
+ "learning_rate": 0.0001361368496079829,
471
+ "loss": 0.1189,
472
+ "step": 3300
473
+ },
474
+ {
475
+ "epoch": 0.6431794182586157,
476
+ "grad_norm": 0.2555278539657593,
477
+ "learning_rate": 0.00013257305773342834,
478
+ "loss": 0.1236,
479
+ "step": 3350
480
+ },
481
+ {
482
+ "epoch": 0.6527791110684458,
483
+ "grad_norm": 0.2080872356891632,
484
+ "learning_rate": 0.00012900926585887384,
485
+ "loss": 0.1187,
486
+ "step": 3400
487
+ },
488
+ {
489
+ "epoch": 0.6623788038782759,
490
+ "grad_norm": 0.27079683542251587,
491
+ "learning_rate": 0.0001254454739843193,
492
+ "loss": 0.1169,
493
+ "step": 3450
494
+ },
495
+ {
496
+ "epoch": 0.6719784966881059,
497
+ "grad_norm": 0.3163507282733917,
498
+ "learning_rate": 0.00012188168210976478,
499
+ "loss": 0.1182,
500
+ "step": 3500
501
+ },
502
+ {
503
+ "epoch": 0.681578189497936,
504
+ "grad_norm": 0.23527826368808746,
505
+ "learning_rate": 0.00011831789023521026,
506
+ "loss": 0.1144,
507
+ "step": 3550
508
+ },
509
+ {
510
+ "epoch": 0.6911778823077661,
511
+ "grad_norm": 0.2670173943042755,
512
+ "learning_rate": 0.00011475409836065573,
513
+ "loss": 0.1133,
514
+ "step": 3600
515
+ },
516
+ {
517
+ "epoch": 0.7007775751175962,
518
+ "grad_norm": 0.22517059743404388,
519
+ "learning_rate": 0.0001111903064861012,
520
+ "loss": 0.1153,
521
+ "step": 3650
522
+ },
523
+ {
524
+ "epoch": 0.7103772679274263,
525
+ "grad_norm": 0.22424179315567017,
526
+ "learning_rate": 0.00010762651461154668,
527
+ "loss": 0.1125,
528
+ "step": 3700
529
+ },
530
+ {
531
+ "epoch": 0.7199769607372564,
532
+ "grad_norm": 0.24241302907466888,
533
+ "learning_rate": 0.00010406272273699216,
534
+ "loss": 0.1132,
535
+ "step": 3750
536
+ },
537
+ {
538
+ "epoch": 0.7295766535470865,
539
+ "grad_norm": 0.3849368393421173,
540
+ "learning_rate": 0.00010049893086243763,
541
+ "loss": 0.1129,
542
+ "step": 3800
543
+ },
544
+ {
545
+ "epoch": 0.7391763463569165,
546
+ "grad_norm": 0.26350146532058716,
547
+ "learning_rate": 9.69351389878831e-05,
548
+ "loss": 0.1098,
549
+ "step": 3850
550
+ },
551
+ {
552
+ "epoch": 0.7487760391667466,
553
+ "grad_norm": 0.15014544129371643,
554
+ "learning_rate": 9.337134711332857e-05,
555
+ "loss": 0.1083,
556
+ "step": 3900
557
+ },
558
+ {
559
+ "epoch": 0.7583757319765767,
560
+ "grad_norm": 0.23375675082206726,
561
+ "learning_rate": 8.980755523877404e-05,
562
+ "loss": 0.1085,
563
+ "step": 3950
564
+ },
565
+ {
566
+ "epoch": 0.7679754247864068,
567
+ "grad_norm": 0.19782401621341705,
568
+ "learning_rate": 8.624376336421953e-05,
569
+ "loss": 0.1088,
570
+ "step": 4000
571
+ },
572
+ {
573
+ "epoch": 0.7775751175962369,
574
+ "grad_norm": 0.1905597746372223,
575
+ "learning_rate": 8.2679971489665e-05,
576
+ "loss": 0.1103,
577
+ "step": 4050
578
+ },
579
+ {
580
+ "epoch": 0.787174810406067,
581
+ "grad_norm": 0.18188077211380005,
582
+ "learning_rate": 7.911617961511047e-05,
583
+ "loss": 0.1074,
584
+ "step": 4100
585
+ },
586
+ {
587
+ "epoch": 0.796774503215897,
588
+ "grad_norm": 0.29012030363082886,
589
+ "learning_rate": 7.555238774055594e-05,
590
+ "loss": 0.1092,
591
+ "step": 4150
592
+ },
593
+ {
594
+ "epoch": 0.8063741960257271,
595
+ "grad_norm": 0.17645134031772614,
596
+ "learning_rate": 7.198859586600141e-05,
597
+ "loss": 0.1067,
598
+ "step": 4200
599
+ },
600
+ {
601
+ "epoch": 0.8159738888355572,
602
+ "grad_norm": 0.17022430896759033,
603
+ "learning_rate": 6.84248039914469e-05,
604
+ "loss": 0.1044,
605
+ "step": 4250
606
+ },
607
+ {
608
+ "epoch": 0.8255735816453873,
609
+ "grad_norm": 0.24264435470104218,
610
+ "learning_rate": 6.486101211689237e-05,
611
+ "loss": 0.1035,
612
+ "step": 4300
613
+ },
614
+ {
615
+ "epoch": 0.8351732744552174,
616
+ "grad_norm": 0.183514803647995,
617
+ "learning_rate": 6.129722024233784e-05,
618
+ "loss": 0.105,
619
+ "step": 4350
620
+ },
621
+ {
622
+ "epoch": 0.8447729672650475,
623
+ "grad_norm": 0.17276549339294434,
624
+ "learning_rate": 5.7733428367783314e-05,
625
+ "loss": 0.1056,
626
+ "step": 4400
627
+ },
628
+ {
629
+ "epoch": 0.8543726600748776,
630
+ "grad_norm": 0.18963700532913208,
631
+ "learning_rate": 5.416963649322879e-05,
632
+ "loss": 0.1044,
633
+ "step": 4450
634
+ },
635
+ {
636
+ "epoch": 0.8639723528847076,
637
+ "grad_norm": 0.15302102267742157,
638
+ "learning_rate": 5.060584461867427e-05,
639
+ "loss": 0.1041,
640
+ "step": 4500
641
+ },
642
+ {
643
+ "epoch": 0.8735720456945377,
644
+ "grad_norm": 0.21309687197208405,
645
+ "learning_rate": 4.7042052744119735e-05,
646
+ "loss": 0.1057,
647
+ "step": 4550
648
+ },
649
+ {
650
+ "epoch": 0.8831717385043678,
651
+ "grad_norm": 0.1691625565290451,
652
+ "learning_rate": 4.3478260869565214e-05,
653
+ "loss": 0.1042,
654
+ "step": 4600
655
+ },
656
+ {
657
+ "epoch": 0.8927714313141979,
658
+ "grad_norm": 0.19566959142684937,
659
+ "learning_rate": 3.9914468995010685e-05,
660
+ "loss": 0.1033,
661
+ "step": 4650
662
+ },
663
+ {
664
+ "epoch": 0.902371124124028,
665
+ "grad_norm": 0.14246854186058044,
666
+ "learning_rate": 3.6350677120456164e-05,
667
+ "loss": 0.103,
668
+ "step": 4700
669
+ },
670
+ {
671
+ "epoch": 0.9119708169338581,
672
+ "grad_norm": 0.14088313281536102,
673
+ "learning_rate": 3.2786885245901635e-05,
674
+ "loss": 0.1026,
675
+ "step": 4750
676
+ },
677
+ {
678
+ "epoch": 0.9215705097436881,
679
+ "grad_norm": 0.15075068175792694,
680
+ "learning_rate": 2.922309337134711e-05,
681
+ "loss": 0.1008,
682
+ "step": 4800
683
+ },
684
+ {
685
+ "epoch": 0.9311702025535182,
686
+ "grad_norm": 0.15009671449661255,
687
+ "learning_rate": 2.5659301496792585e-05,
688
+ "loss": 0.1043,
689
+ "step": 4850
690
+ },
691
+ {
692
+ "epoch": 0.9407698953633483,
693
+ "grad_norm": 0.2823892831802368,
694
+ "learning_rate": 2.209550962223806e-05,
695
+ "loss": 0.1013,
696
+ "step": 4900
697
+ },
698
+ {
699
+ "epoch": 0.9503695881731784,
700
+ "grad_norm": 0.15409894287586212,
701
+ "learning_rate": 1.8531717747683532e-05,
702
+ "loss": 0.1024,
703
+ "step": 4950
704
+ },
705
+ {
706
+ "epoch": 0.9599692809830085,
707
+ "grad_norm": 0.15414948761463165,
708
+ "learning_rate": 1.4967925873129009e-05,
709
+ "loss": 0.0996,
710
+ "step": 5000
711
+ },
712
+ {
713
+ "epoch": 0.9695689737928387,
714
+ "grad_norm": 0.1376316249370575,
715
+ "learning_rate": 1.1404133998574482e-05,
716
+ "loss": 0.1026,
717
+ "step": 5050
718
+ },
719
+ {
720
+ "epoch": 0.9791686666026688,
721
+ "grad_norm": 0.12536156177520752,
722
+ "learning_rate": 7.840342124019957e-06,
723
+ "loss": 0.1018,
724
+ "step": 5100
725
+ },
726
+ {
727
+ "epoch": 0.9887683594124989,
728
+ "grad_norm": 0.1290249526500702,
729
+ "learning_rate": 4.276550249465431e-06,
730
+ "loss": 0.1038,
731
+ "step": 5150
732
+ },
733
+ {
734
+ "epoch": 0.9983680522223289,
735
+ "grad_norm": 0.14181403815746307,
736
+ "learning_rate": 7.127583749109051e-07,
737
+ "loss": 0.1025,
738
+ "step": 5200
739
+ }
740
+ ],
741
+ "logging_steps": 50,
742
+ "max_steps": 5209,
743
+ "num_input_tokens_seen": 0,
744
+ "num_train_epochs": 1,
745
+ "save_steps": 2000,
746
+ "stateful_callbacks": {
747
+ "TrainerControl": {
748
+ "args": {
749
+ "should_epoch_stop": false,
750
+ "should_evaluate": false,
751
+ "should_log": false,
752
+ "should_save": true,
753
+ "should_training_stop": true
754
+ },
755
+ "attributes": {}
756
+ }
757
+ },
758
+ "total_flos": 1.1341819211769446e+17,
759
+ "train_batch_size": 16,
760
+ "trial_name": null,
761
+ "trial_params": null
762
+ }
checkpoint-5209/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:30878ac8e72b88c8cbfce7348f711b9b45f86117719d802abb44671778afc8df
3
+ size 5841
config.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "MLAForCausalLM"
4
+ ],
5
+ "bos_token_id": 0,
6
+ "dtype": "float32",
7
+ "elementwise_affine": true,
8
+ "eos_token_id": 1,
9
+ "fuse_cross_entropy": true,
10
+ "fuse_linear_cross_entropy": false,
11
+ "fuse_norm": true,
12
+ "fuse_swiglu": true,
13
+ "hidden_act": "swish",
14
+ "hidden_ratio": 4,
15
+ "hidden_size": 256,
16
+ "initializer_range": 0.02,
17
+ "intermediate_size": 896,
18
+ "kv_lora_rank": 64,
19
+ "max_position_embeddings": 2048,
20
+ "model_type": "mla",
21
+ "norm_eps": 1e-05,
22
+ "num_heads": 4,
23
+ "num_hidden_layers": 35,
24
+ "q_lora_rank": 32,
25
+ "qk_head_dim": 96,
26
+ "qk_nope_head_dim": 64,
27
+ "qk_rope_head_dim": 32,
28
+ "rope_scaling": null,
29
+ "rope_theta": 10000.0,
30
+ "tie_word_embeddings": true,
31
+ "transformers_version": "4.57.1",
32
+ "use_cache": true,
33
+ "use_l2warp": false,
34
+ "v_head_dim": 32,
35
+ "vocab_size": 256,
36
+ "window_size": null
37
+ }
generation_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 0,
4
+ "eos_token_id": 1,
5
+ "transformers_version": "4.57.1"
6
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c0b0e02995fa075d7e0662517e14fd69461add8b38dafd18e07bb4d4037fbc98
3
+ size 111070184
special_tokens_map.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<think>",
4
+ "</think>"
5
+ ],
6
+ "bos_token": {
7
+ "content": "<|startoftext|>",
8
+ "lstrip": false,
9
+ "normalized": false,
10
+ "rstrip": false,
11
+ "single_word": false
12
+ },
13
+ "eos_token": {
14
+ "content": "<|endoftext|>",
15
+ "lstrip": false,
16
+ "normalized": false,
17
+ "rstrip": false,
18
+ "single_word": false
19
+ },
20
+ "pad_token": "<|pad|>",
21
+ "unk_token": {
22
+ "content": "<|unknown|>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false
27
+ }
28
+ }
tokenizer.json ADDED
@@ -0,0 +1,1085 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "version": "1.0",
3
+ "truncation": null,
4
+ "padding": null,
5
+ "added_tokens": [
6
+ {
7
+ "id": 0,
8
+ "content": "<|startoftext|>",
9
+ "single_word": false,
10
+ "lstrip": false,
11
+ "rstrip": false,
12
+ "normalized": false,
13
+ "special": true
14
+ },
15
+ {
16
+ "id": 1,
17
+ "content": "<|endoftext|>",
18
+ "single_word": false,
19
+ "lstrip": false,
20
+ "rstrip": false,
21
+ "normalized": false,
22
+ "special": true
23
+ },
24
+ {
25
+ "id": 2,
26
+ "content": "<|unknown|>",
27
+ "single_word": false,
28
+ "lstrip": false,
29
+ "rstrip": false,
30
+ "normalized": false,
31
+ "special": true
32
+ },
33
+ {
34
+ "id": 3,
35
+ "content": "<|pad|>",
36
+ "single_word": false,
37
+ "lstrip": false,
38
+ "rstrip": false,
39
+ "normalized": false,
40
+ "special": true
41
+ },
42
+ {
43
+ "id": 4,
44
+ "content": "<think>",
45
+ "single_word": false,
46
+ "lstrip": false,
47
+ "rstrip": false,
48
+ "normalized": false,
49
+ "special": true
50
+ },
51
+ {
52
+ "id": 5,
53
+ "content": "</think>",
54
+ "single_word": false,
55
+ "lstrip": false,
56
+ "rstrip": false,
57
+ "normalized": false,
58
+ "special": true
59
+ }
60
+ ],
61
+ "normalizer": null,
62
+ "pre_tokenizer": {
63
+ "type": "ByteLevel",
64
+ "add_prefix_space": false,
65
+ "trim_offsets": true,
66
+ "use_regex": true
67
+ },
68
+ "post_processor": null,
69
+ "decoder": {
70
+ "type": "ByteLevel",
71
+ "add_prefix_space": true,
72
+ "trim_offsets": true,
73
+ "use_regex": true
74
+ },
75
+ "model": {
76
+ "type": "BPE",
77
+ "dropout": null,
78
+ "unk_token": null,
79
+ "continuing_subword_prefix": null,
80
+ "end_of_word_suffix": null,
81
+ "fuse_unk": false,
82
+ "byte_fallback": false,
83
+ "ignore_merges": false,
84
+ "vocab": {
85
+ "<|startoftext|>": 0,
86
+ "<|endoftext|>": 1,
87
+ "<|unknown|>": 2,
88
+ "<|pad|>": 3,
89
+ "<think>": 4,
90
+ "</think>": 5,
91
+ "%": 6,
92
+ "&": 7,
93
+ "'": 8,
94
+ "(": 9,
95
+ ")": 10,
96
+ ",": 11,
97
+ "-": 12,
98
+ ".": 13,
99
+ "0": 14,
100
+ "1": 15,
101
+ "2": 16,
102
+ "3": 17,
103
+ "4": 18,
104
+ "5": 19,
105
+ "6": 20,
106
+ "7": 21,
107
+ "8": 22,
108
+ "9": 23,
109
+ ":": 24,
110
+ ">": 25,
111
+ "A": 26,
112
+ "C": 27,
113
+ "D": 28,
114
+ "E": 29,
115
+ "F": 30,
116
+ "G": 31,
117
+ "H": 32,
118
+ "I": 33,
119
+ "L": 34,
120
+ "M": 35,
121
+ "N": 36,
122
+ "O": 37,
123
+ "P": 38,
124
+ "R": 39,
125
+ "S": 40,
126
+ "T": 41,
127
+ "V": 42,
128
+ "[": 43,
129
+ "]": 44,
130
+ "a": 45,
131
+ "b": 46,
132
+ "c": 47,
133
+ "d": 48,
134
+ "e": 49,
135
+ "f": 50,
136
+ "g": 51,
137
+ "h": 52,
138
+ "i": 53,
139
+ "k": 54,
140
+ "l": 55,
141
+ "m": 56,
142
+ "n": 57,
143
+ "o": 58,
144
+ "p": 59,
145
+ "q": 60,
146
+ "r": 61,
147
+ "s": 62,
148
+ "t": 63,
149
+ "u": 64,
150
+ "v": 65,
151
+ "x": 66,
152
+ "y": 67,
153
+ "|": 68,
154
+ "Ċ": 69,
155
+ "Ġ": 70,
156
+ "00": 71,
157
+ "11": 72,
158
+ "->": 73,
159
+ "Ġ|": 74,
160
+ ":[": 75,
161
+ "]->": 76,
162
+ "10": 77,
163
+ "Ġ1": 78,
164
+ "000": 79,
165
+ "ĠS": 80,
166
+ "',": 81,
167
+ "Ġ'": 82,
168
+ "te": 83,
169
+ "100": 84,
170
+ "ĊĠ": 85,
171
+ "110": 86,
172
+ "tep": 87,
173
+ "re": 88,
174
+ "111": 89,
175
+ "0000": 90,
176
+ "011": 91,
177
+ "101": 92,
178
+ "001": 93,
179
+ "ĠStep": 94,
180
+ "010": 95,
181
+ "in": 96,
182
+ "im": 97,
183
+ "ĠSim": 98,
184
+ "1111": 99,
185
+ "Ġ[": 100,
186
+ "or": 101,
187
+ "ĠI": 102,
188
+ "ho": 103,
189
+ "od": 104,
190
+ "hood": 105,
191
+ "Ġin": 106,
192
+ "Ġ11": 107,
193
+ "Ġ10": 108,
194
+ "ne": 109,
195
+ "Ġne": 110,
196
+ "']": 111,
197
+ "bor": 112,
198
+ "dx": 113,
199
+ "gh": 114,
200
+ "ire": 115,
201
+ "igh": 116,
202
+ "qu": 117,
203
+ "Ġre": 118,
204
+ "Ġ['": 119,
205
+ "ĠIdx": 120,
206
+ "Ġneigh": 121,
207
+ "borhood": 122,
208
+ "ires": 123,
209
+ "quires": 124,
210
+ "Ġrequires": 125,
211
+ "Ġneighborhood": 126,
212
+ "Step": 127,
213
+ "ut": 128,
214
+ "Ġ3": 129,
215
+ "Ġ2": 130,
216
+ "Ġ4": 131,
217
+ "Ġ5": 132,
218
+ "Ġ6": 133,
219
+ "Ġ9": 134,
220
+ "Ġ8": 135,
221
+ "Ġ7": 136,
222
+ "Ġ0": 137,
223
+ "ca": 138,
224
+ "Ġ12": 139,
225
+ "Ġ13": 140,
226
+ "1010": 141,
227
+ "Ġ14": 142,
228
+ "Out": 143,
229
+ "put": 144,
230
+ "ĠOut": 145,
231
+ "ĠSca": 146,
232
+ "ĠOutput": 147,
233
+ "ĠScan": 148,
234
+ "Ġ011": 149,
235
+ "ing": 150,
236
+ "ti": 151,
237
+ "1100": 152,
238
+ "00000000": 153,
239
+ "Ġ15": 154,
240
+ "Ġ000": 155,
241
+ "on": 156,
242
+ "Ġ110": 157,
243
+ "ul": 158,
244
+ "Ġ100": 159,
245
+ "at": 160,
246
+ "tion": 161,
247
+ "Ġ010": 162,
248
+ "1011": 163,
249
+ "ĠC": 164,
250
+ "Ġ00": 165,
251
+ "00011": 166,
252
+ "50": 167,
253
+ "er": 168,
254
+ "10000": 169,
255
+ "nd": 170,
256
+ "Ġ16": 171,
257
+ "ule": 172,
258
+ "ĠR": 173,
259
+ "ed": 174,
260
+ "li": 175,
261
+ "Ġ001": 176,
262
+ "Ġ101": 177,
263
+ "ate": 178,
264
+ "ĠRule": 179,
265
+ "1000": 180,
266
+ "fi": 181,
267
+ "10100": 182,
268
+ "as": 183,
269
+ "Ġ17": 184,
270
+ "ĠM": 185,
271
+ "Ġ1111": 186,
272
+ "Ver": 187,
273
+ "ch": 188,
274
+ "es": 189,
275
+ "ifi": 190,
276
+ "mp": 191,
277
+ "ting": 192,
278
+ "cation": 193,
279
+ "atch": 194,
280
+ "Verifi": 195,
281
+ "Verification": 196,
282
+ "..": 197,
283
+ "11011": 198,
284
+ "41": 199,
285
+ "and": 200,
286
+ "date": 201,
287
+ "idate": 202,
288
+ "ĠCand": 203,
289
+ "ĠCandidate": 204,
290
+ "58": 205,
291
+ "ating": 206,
292
+ "10011": 207,
293
+ "ction": 208,
294
+ "111000": 209,
295
+ "000000": 210,
296
+ "10110": 211,
297
+ "Ġ00000000": 212,
298
+ "is": 213,
299
+ "match": 214,
300
+ "ĠMis": 215,
301
+ "ĠMismatch": 216,
302
+ "111111": 217,
303
+ "Tas": 218,
304
+ "Task": 219,
305
+ "Ġ18": 220,
306
+ "Gre": 221,
307
+ "Pr": 222,
308
+ "ĠL": 223,
309
+ "ĠGre": 224,
310
+ "ĠPr": 225,
311
+ "edy": 226,
312
+ "ĠGreedy": 227,
313
+ "At": 228,
314
+ "Ex": 229,
315
+ "Tes": 230,
316
+ "ĠTes": 231,
317
+ "temp": 232,
318
+ "Attemp": 233,
319
+ "ĠTesting": 234,
320
+ "Attempt": 235,
321
+ "100000": 236,
322
+ "Ġs": 237,
323
+ "Ġ0000": 238,
324
+ "uction": 239,
325
+ "ĠSteps": 240,
326
+ "xt": 241,
327
+ "101111": 242,
328
+ "al": 243,
329
+ "cul": 244,
330
+ "eli": 245,
331
+ "ior": 246,
332
+ "keli": 247,
333
+ "ver": 248,
334
+ "cal": 249,
335
+ "culating": 250,
336
+ "iors": 251,
337
+ "kelihood": 252,
338
+ "Ġ19": 253,
339
+ "11111111": 254,
340
+ ").": 255
341
+ },
342
+ "merges": [
343
+ [
344
+ "0",
345
+ "0"
346
+ ],
347
+ [
348
+ "1",
349
+ "1"
350
+ ],
351
+ [
352
+ "-",
353
+ ">"
354
+ ],
355
+ [
356
+ "Ġ",
357
+ "|"
358
+ ],
359
+ [
360
+ ":",
361
+ "["
362
+ ],
363
+ [
364
+ "]",
365
+ "->"
366
+ ],
367
+ [
368
+ "1",
369
+ "0"
370
+ ],
371
+ [
372
+ "Ġ",
373
+ "1"
374
+ ],
375
+ [
376
+ "00",
377
+ "0"
378
+ ],
379
+ [
380
+ "Ġ",
381
+ "S"
382
+ ],
383
+ [
384
+ "'",
385
+ ","
386
+ ],
387
+ [
388
+ "Ġ",
389
+ "'"
390
+ ],
391
+ [
392
+ "t",
393
+ "e"
394
+ ],
395
+ [
396
+ "1",
397
+ "00"
398
+ ],
399
+ [
400
+ "Ċ",
401
+ "Ġ"
402
+ ],
403
+ [
404
+ "11",
405
+ "0"
406
+ ],
407
+ [
408
+ "te",
409
+ "p"
410
+ ],
411
+ [
412
+ "r",
413
+ "e"
414
+ ],
415
+ [
416
+ "11",
417
+ "1"
418
+ ],
419
+ [
420
+ "00",
421
+ "00"
422
+ ],
423
+ [
424
+ "0",
425
+ "11"
426
+ ],
427
+ [
428
+ "10",
429
+ "1"
430
+ ],
431
+ [
432
+ "00",
433
+ "1"
434
+ ],
435
+ [
436
+ "ĠS",
437
+ "tep"
438
+ ],
439
+ [
440
+ "0",
441
+ "10"
442
+ ],
443
+ [
444
+ "i",
445
+ "n"
446
+ ],
447
+ [
448
+ "i",
449
+ "m"
450
+ ],
451
+ [
452
+ "ĠS",
453
+ "im"
454
+ ],
455
+ [
456
+ "11",
457
+ "11"
458
+ ],
459
+ [
460
+ "Ġ",
461
+ "["
462
+ ],
463
+ [
464
+ "o",
465
+ "r"
466
+ ],
467
+ [
468
+ "Ġ",
469
+ "I"
470
+ ],
471
+ [
472
+ "h",
473
+ "o"
474
+ ],
475
+ [
476
+ "o",
477
+ "d"
478
+ ],
479
+ [
480
+ "ho",
481
+ "od"
482
+ ],
483
+ [
484
+ "Ġ",
485
+ "in"
486
+ ],
487
+ [
488
+ "Ġ",
489
+ "11"
490
+ ],
491
+ [
492
+ "Ġ",
493
+ "10"
494
+ ],
495
+ [
496
+ "n",
497
+ "e"
498
+ ],
499
+ [
500
+ "Ġ",
501
+ "ne"
502
+ ],
503
+ [
504
+ "'",
505
+ "]"
506
+ ],
507
+ [
508
+ "b",
509
+ "or"
510
+ ],
511
+ [
512
+ "d",
513
+ "x"
514
+ ],
515
+ [
516
+ "g",
517
+ "h"
518
+ ],
519
+ [
520
+ "i",
521
+ "re"
522
+ ],
523
+ [
524
+ "i",
525
+ "gh"
526
+ ],
527
+ [
528
+ "q",
529
+ "u"
530
+ ],
531
+ [
532
+ "Ġ",
533
+ "re"
534
+ ],
535
+ [
536
+ "Ġ[",
537
+ "'"
538
+ ],
539
+ [
540
+ "ĠI",
541
+ "dx"
542
+ ],
543
+ [
544
+ "Ġne",
545
+ "igh"
546
+ ],
547
+ [
548
+ "bor",
549
+ "hood"
550
+ ],
551
+ [
552
+ "ire",
553
+ "s"
554
+ ],
555
+ [
556
+ "qu",
557
+ "ires"
558
+ ],
559
+ [
560
+ "Ġre",
561
+ "quires"
562
+ ],
563
+ [
564
+ "Ġneigh",
565
+ "borhood"
566
+ ],
567
+ [
568
+ "S",
569
+ "tep"
570
+ ],
571
+ [
572
+ "u",
573
+ "t"
574
+ ],
575
+ [
576
+ "Ġ",
577
+ "3"
578
+ ],
579
+ [
580
+ "Ġ",
581
+ "2"
582
+ ],
583
+ [
584
+ "Ġ",
585
+ "4"
586
+ ],
587
+ [
588
+ "Ġ",
589
+ "5"
590
+ ],
591
+ [
592
+ "Ġ",
593
+ "6"
594
+ ],
595
+ [
596
+ "Ġ",
597
+ "9"
598
+ ],
599
+ [
600
+ "Ġ",
601
+ "8"
602
+ ],
603
+ [
604
+ "Ġ",
605
+ "7"
606
+ ],
607
+ [
608
+ "Ġ",
609
+ "0"
610
+ ],
611
+ [
612
+ "c",
613
+ "a"
614
+ ],
615
+ [
616
+ "Ġ1",
617
+ "2"
618
+ ],
619
+ [
620
+ "Ġ1",
621
+ "3"
622
+ ],
623
+ [
624
+ "10",
625
+ "10"
626
+ ],
627
+ [
628
+ "Ġ1",
629
+ "4"
630
+ ],
631
+ [
632
+ "O",
633
+ "ut"
634
+ ],
635
+ [
636
+ "p",
637
+ "ut"
638
+ ],
639
+ [
640
+ "Ġ",
641
+ "Out"
642
+ ],
643
+ [
644
+ "ĠS",
645
+ "ca"
646
+ ],
647
+ [
648
+ "ĠOut",
649
+ "put"
650
+ ],
651
+ [
652
+ "ĠSca",
653
+ "n"
654
+ ],
655
+ [
656
+ "Ġ",
657
+ "011"
658
+ ],
659
+ [
660
+ "in",
661
+ "g"
662
+ ],
663
+ [
664
+ "t",
665
+ "i"
666
+ ],
667
+ [
668
+ "11",
669
+ "00"
670
+ ],
671
+ [
672
+ "0000",
673
+ "0000"
674
+ ],
675
+ [
676
+ "Ġ1",
677
+ "5"
678
+ ],
679
+ [
680
+ "Ġ",
681
+ "000"
682
+ ],
683
+ [
684
+ "o",
685
+ "n"
686
+ ],
687
+ [
688
+ "Ġ",
689
+ "110"
690
+ ],
691
+ [
692
+ "u",
693
+ "l"
694
+ ],
695
+ [
696
+ "Ġ1",
697
+ "00"
698
+ ],
699
+ [
700
+ "a",
701
+ "t"
702
+ ],
703
+ [
704
+ "ti",
705
+ "on"
706
+ ],
707
+ [
708
+ "Ġ",
709
+ "010"
710
+ ],
711
+ [
712
+ "10",
713
+ "11"
714
+ ],
715
+ [
716
+ "Ġ",
717
+ "C"
718
+ ],
719
+ [
720
+ "Ġ",
721
+ "00"
722
+ ],
723
+ [
724
+ "000",
725
+ "11"
726
+ ],
727
+ [
728
+ "5",
729
+ "0"
730
+ ],
731
+ [
732
+ "e",
733
+ "r"
734
+ ],
735
+ [
736
+ "100",
737
+ "00"
738
+ ],
739
+ [
740
+ "n",
741
+ "d"
742
+ ],
743
+ [
744
+ "Ġ1",
745
+ "6"
746
+ ],
747
+ [
748
+ "ul",
749
+ "e"
750
+ ],
751
+ [
752
+ "Ġ",
753
+ "R"
754
+ ],
755
+ [
756
+ "e",
757
+ "d"
758
+ ],
759
+ [
760
+ "l",
761
+ "i"
762
+ ],
763
+ [
764
+ "Ġ",
765
+ "001"
766
+ ],
767
+ [
768
+ "Ġ",
769
+ "101"
770
+ ],
771
+ [
772
+ "a",
773
+ "te"
774
+ ],
775
+ [
776
+ "ĠR",
777
+ "ule"
778
+ ],
779
+ [
780
+ "1",
781
+ "000"
782
+ ],
783
+ [
784
+ "f",
785
+ "i"
786
+ ],
787
+ [
788
+ "10",
789
+ "100"
790
+ ],
791
+ [
792
+ "a",
793
+ "s"
794
+ ],
795
+ [
796
+ "Ġ1",
797
+ "7"
798
+ ],
799
+ [
800
+ "Ġ",
801
+ "M"
802
+ ],
803
+ [
804
+ "Ġ",
805
+ "1111"
806
+ ],
807
+ [
808
+ "V",
809
+ "er"
810
+ ],
811
+ [
812
+ "c",
813
+ "h"
814
+ ],
815
+ [
816
+ "e",
817
+ "s"
818
+ ],
819
+ [
820
+ "i",
821
+ "fi"
822
+ ],
823
+ [
824
+ "m",
825
+ "p"
826
+ ],
827
+ [
828
+ "t",
829
+ "ing"
830
+ ],
831
+ [
832
+ "ca",
833
+ "tion"
834
+ ],
835
+ [
836
+ "at",
837
+ "ch"
838
+ ],
839
+ [
840
+ "Ver",
841
+ "ifi"
842
+ ],
843
+ [
844
+ "Verifi",
845
+ "cation"
846
+ ],
847
+ [
848
+ ".",
849
+ "."
850
+ ],
851
+ [
852
+ "110",
853
+ "11"
854
+ ],
855
+ [
856
+ "4",
857
+ "1"
858
+ ],
859
+ [
860
+ "a",
861
+ "nd"
862
+ ],
863
+ [
864
+ "d",
865
+ "ate"
866
+ ],
867
+ [
868
+ "i",
869
+ "date"
870
+ ],
871
+ [
872
+ "ĠC",
873
+ "and"
874
+ ],
875
+ [
876
+ "ĠCand",
877
+ "idate"
878
+ ],
879
+ [
880
+ "5",
881
+ "8"
882
+ ],
883
+ [
884
+ "at",
885
+ "ing"
886
+ ],
887
+ [
888
+ "100",
889
+ "11"
890
+ ],
891
+ [
892
+ "c",
893
+ "tion"
894
+ ],
895
+ [
896
+ "111",
897
+ "000"
898
+ ],
899
+ [
900
+ "0000",
901
+ "00"
902
+ ],
903
+ [
904
+ "10",
905
+ "110"
906
+ ],
907
+ [
908
+ "Ġ",
909
+ "00000000"
910
+ ],
911
+ [
912
+ "i",
913
+ "s"
914
+ ],
915
+ [
916
+ "m",
917
+ "atch"
918
+ ],
919
+ [
920
+ "ĠM",
921
+ "is"
922
+ ],
923
+ [
924
+ "ĠMis",
925
+ "match"
926
+ ],
927
+ [
928
+ "1111",
929
+ "11"
930
+ ],
931
+ [
932
+ "T",
933
+ "as"
934
+ ],
935
+ [
936
+ "Tas",
937
+ "k"
938
+ ],
939
+ [
940
+ "Ġ1",
941
+ "8"
942
+ ],
943
+ [
944
+ "G",
945
+ "re"
946
+ ],
947
+ [
948
+ "P",
949
+ "r"
950
+ ],
951
+ [
952
+ "Ġ",
953
+ "L"
954
+ ],
955
+ [
956
+ "Ġ",
957
+ "Gre"
958
+ ],
959
+ [
960
+ "Ġ",
961
+ "Pr"
962
+ ],
963
+ [
964
+ "ed",
965
+ "y"
966
+ ],
967
+ [
968
+ "ĠGre",
969
+ "edy"
970
+ ],
971
+ [
972
+ "A",
973
+ "t"
974
+ ],
975
+ [
976
+ "E",
977
+ "x"
978
+ ],
979
+ [
980
+ "T",
981
+ "es"
982
+ ],
983
+ [
984
+ "Ġ",
985
+ "Tes"
986
+ ],
987
+ [
988
+ "te",
989
+ "mp"
990
+ ],
991
+ [
992
+ "At",
993
+ "temp"
994
+ ],
995
+ [
996
+ "ĠTes",
997
+ "ting"
998
+ ],
999
+ [
1000
+ "Attemp",
1001
+ "t"
1002
+ ],
1003
+ [
1004
+ "100",
1005
+ "000"
1006
+ ],
1007
+ [
1008
+ "Ġ",
1009
+ "s"
1010
+ ],
1011
+ [
1012
+ "Ġ",
1013
+ "0000"
1014
+ ],
1015
+ [
1016
+ "u",
1017
+ "ction"
1018
+ ],
1019
+ [
1020
+ "ĠStep",
1021
+ "s"
1022
+ ],
1023
+ [
1024
+ "x",
1025
+ "t"
1026
+ ],
1027
+ [
1028
+ "10",
1029
+ "1111"
1030
+ ],
1031
+ [
1032
+ "a",
1033
+ "l"
1034
+ ],
1035
+ [
1036
+ "c",
1037
+ "ul"
1038
+ ],
1039
+ [
1040
+ "e",
1041
+ "li"
1042
+ ],
1043
+ [
1044
+ "i",
1045
+ "or"
1046
+ ],
1047
+ [
1048
+ "k",
1049
+ "eli"
1050
+ ],
1051
+ [
1052
+ "v",
1053
+ "er"
1054
+ ],
1055
+ [
1056
+ "ca",
1057
+ "l"
1058
+ ],
1059
+ [
1060
+ "cul",
1061
+ "ating"
1062
+ ],
1063
+ [
1064
+ "ior",
1065
+ "s"
1066
+ ],
1067
+ [
1068
+ "keli",
1069
+ "hood"
1070
+ ],
1071
+ [
1072
+ "Ġ1",
1073
+ "9"
1074
+ ],
1075
+ [
1076
+ "1111",
1077
+ "1111"
1078
+ ],
1079
+ [
1080
+ ")",
1081
+ "."
1082
+ ]
1083
+ ]
1084
+ }
1085
+ }
tokenizer_config.json ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<|startoftext|>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "<|endoftext|>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "<|unknown|>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "3": {
28
+ "content": "<|pad|>",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "4": {
36
+ "content": "<think>",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ },
43
+ "5": {
44
+ "content": "</think>",
45
+ "lstrip": false,
46
+ "normalized": false,
47
+ "rstrip": false,
48
+ "single_word": false,
49
+ "special": true
50
+ }
51
+ },
52
+ "additional_special_tokens": [
53
+ "<think>",
54
+ "</think>"
55
+ ],
56
+ "bos_token": "<|startoftext|>",
57
+ "clean_up_tokenization_spaces": false,
58
+ "eos_token": "<|endoftext|>",
59
+ "extra_special_tokens": {},
60
+ "model_max_length": 1000000000000000019884624838656,
61
+ "pad_token": "<|pad|>",
62
+ "tokenizer_class": "PreTrainedTokenizerFast",
63
+ "unk_token": "<|unknown|>"
64
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:30878ac8e72b88c8cbfce7348f711b9b45f86117719d802abb44671778afc8df
3
+ size 5841