Hugging Face's logo

PaddlePaddle
/
PP-Chart2Table_safetensors

Image-to-Text
PaddleOCR
Safetensors
English
Chinese
pp_chart2table
OCR
PaddlePaddle
chart_parsing
custom_code
Model card Files
xet
 Community
2
xiaohei66 commited on
Commit
dce71b1
·
verified ·
1 Parent(s): fc21169

Upload folder using huggingface_hub

Browse files
Files changed (6) hide show
  1. .gitattributes +1 -0
  2. chat_template.jinja +28 -54
  3. config.json +1 -5
  4. model.safetensors +2 -2
  5. tokenizer.json +3 -0
  6. tokenizer_config.json +5 -1769
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
chat_template.jinja CHANGED
@@ -1,54 +1,28 @@
1
- {%- if tools %}
2
- {{- '<|im_start|>system\n' }}
3
- {%- if messages[0]['role'] == 'system' %}
4
- {{- messages[0]['content'] }}
5
- {%- else %}
6
- {{- 'You are a helpful assistant.' }}
7
- {%- endif %}
8
- {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
9
- {%- for tool in tools %}
10
- {{- "\n" }}
11
- {{- tool | tojson }}
12
- {%- endfor %}
13
- {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
14
- {%- else %}
15
- {%- if messages[0]['role'] == 'system' %}
16
- {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
17
- {%- else %}
18
- {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}
19
- {%- endif %}
20
- {%- endif %}
21
- {%- for message in messages %}
22
- {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
23
- {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
24
- {%- elif message.role == "assistant" %}
25
- {{- '<|im_start|>' + message.role }}
26
- {%- if message.content %}
27
- {{- '\n' + message.content }}
28
- {%- endif %}
29
- {%- for tool_call in message.tool_calls %}
30
- {%- if tool_call.function is defined %}
31
- {%- set tool_call = tool_call.function %}
32
- {%- endif %}
33
- {{- '\n<tool_call>\n{"name": "' }}
34
- {{- tool_call.name }}
35
- {{- '", "arguments": ' }}
36
- {{- tool_call.arguments | tojson }}
37
- {{- '}\n</tool_call>' }}
38
- {%- endfor %}
39
- {{- '<|im_end|>\n' }}
40
- {%- elif message.role == "tool" %}
41
- {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
42
- {{- '<|im_start|>user' }}
43
- {%- endif %}
44
- {{- '\n<tool_response>\n' }}
45
- {{- message.content }}
46
- {{- '\n</tool_response>' }}
47
- {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
48
- {{- '<|im_end|>\n' }}
49
- {%- endif %}
50
- {%- endif %}
51
- {%- endfor %}
52
- {%- if add_generation_prompt %}
53
- {{- '<|im_start|>assistant\n' }}
54
- {%- endif %}
 
1
+ {% set system_query = 'You should follow the instructions carefully and explain your answers in detail.' %}
2
+ {% set default_query = 'Chart to table' %}
3
+ {%- for message in messages -%}
4
+ {%- if message['role'] == 'system' -%}
5
+ <|im_start|>system\n{{- message.get('content', system_query) -}}<|im_end|>
6
+ {%- elif message['role'] == 'user' -%}
7
+ <|im_start|>user\n
8
+ {%- if 'image' in message -%}
9
+ <img>
10
+ {%- set num_pad = message['image']['num_patches'] * message['image']['num_patches'] -%}
11
+ {%- for i in range(num_pad) -%}
12
+ <imgpad>
13
+ {%- endfor -%}
14
+ </img>\n
15
+ {%- endif -%}
16
+ {%- set query = message.get('content', default_query) -%}
17
+ {{- query -}}
18
+ <|im_end|>
19
+ {%- elif message['role'] == 'assistant' -%}
20
+ <|im_start|>assistant\n
21
+ {%- if message.get('content') -%}
22
+ {{- message['content'] -}}
23
+ {%- endif -%}
24
+ {%- endif -%}
25
+ {%- endfor -%}
26
+ {%- if add_generation_prompt -%}
27
+ <|im_start|>assistant\n
28
+ {%- endif -%}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
config.json CHANGED
@@ -1,8 +1,6 @@
1
  {
2
  "model_type": "pp_chart2table",
3
- "architectures": [
4
- "PPChart2TableForConditionalGeneration"
5
- ],
6
  "auto_map": {
7
  "AutoConfig": "configuration_pp_chart2table.PPChart2TableConfig",
8
  "AutoModel": "modeling_pp_chart2table.PPChart2TableForConditionalGeneration"
@@ -22,8 +20,6 @@
22
  "dtype": "bfloat16",
23
  "use_cache": true,
24
  "use_sliding_window": false,
25
- "output_channels": 1024,
26
- "vision_hidden_channels": 512,
27
  "vision_config": {
28
  "architectures": [
29
  "PPChart2TableVisionModel"
 
1
  {
2
  "model_type": "pp_chart2table",
3
+ "architectures": ["GotOcr2ForConditionalGeneration"],
 
 
4
  "auto_map": {
5
  "AutoConfig": "configuration_pp_chart2table.PPChart2TableConfig",
6
  "AutoModel": "modeling_pp_chart2table.PPChart2TableForConditionalGeneration"
 
20
  "dtype": "bfloat16",
21
  "use_cache": true,
22
  "use_sliding_window": false,
 
 
23
  "vision_config": {
24
  "architectures": [
25
  "PPChart2TableVisionModel"
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:fe6b50bbd1f9bc6687a60d1ede46c3ad043859c41c408b555c1850dad0d118b4
3
- size 2242172768
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:725fec36a72e44cf5e1b5e8f1055b1a7d015b730919809a5cfa352f56b4d60bd
3
+ size 2242172856
tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:804a00c21a315623fe96133d70eef3d2bd091bef6579b3ef16ce0e946b555f88
3
+ size 11459023
tokenizer_config.json CHANGED
@@ -1,1774 +1,10 @@
1
  {
2
- "add_bos_token": false,
3
- "add_prefix_space": false,
4
- "added_tokens_decoder": {
5
- "151643": {
6
- "content": "<|endoftext|>",
7
- "lstrip": false,
8
- "normalized": false,
9
- "rstrip": false,
10
- "single_word": false,
11
- "special": true
12
- },
13
- "151644": {
14
- "content": "<|im_start|>",
15
- "lstrip": false,
16
- "normalized": false,
17
- "rstrip": false,
18
- "single_word": false,
19
- "special": true
20
- },
21
- "151645": {
22
- "content": "<|im_end|>",
23
- "lstrip": false,
24
- "normalized": false,
25
- "rstrip": false,
26
- "single_word": false,
27
- "special": true
28
- },
29
- "151646": {
30
- "content": "<|extra_0|>",
31
- "lstrip": false,
32
- "normalized": false,
33
- "rstrip": false,
34
- "single_word": false,
35
- "special": true
36
- },
37
- "151647": {
38
- "content": "<|extra_1|>",
39
- "lstrip": false,
40
- "normalized": false,
41
- "rstrip": false,
42
- "single_word": false,
43
- "special": true
44
- },
45
- "151648": {
46
- "content": "<|extra_2|>",
47
- "lstrip": false,
48
- "normalized": false,
49
- "rstrip": false,
50
- "single_word": false,
51
- "special": true
52
- },
53
- "151649": {
54
- "content": "<|extra_3|>",
55
- "lstrip": false,
56
- "normalized": false,
57
- "rstrip": false,
58
- "single_word": false,
59
- "special": true
60
- },
61
- "151650": {
62
- "content": "<|extra_4|>",
63
- "lstrip": false,
64
- "normalized": false,
65
- "rstrip": false,
66
- "single_word": false,
67
- "special": true
68
- },
69
- "151651": {
70
- "content": "<|extra_5|>",
71
- "lstrip": false,
72
- "normalized": false,
73
- "rstrip": false,
74
- "single_word": false,
75
- "special": true
76
- },
77
- "151652": {
78
- "content": "<|extra_6|>",
79
- "lstrip": false,
80
- "normalized": false,
81
- "rstrip": false,
82
- "single_word": false,
83
- "special": true
84
- },
85
- "151653": {
86
- "content": "<|extra_7|>",
87
- "lstrip": false,
88
- "normalized": false,
89
- "rstrip": false,
90
- "single_word": false,
91
- "special": true
92
- },
93
- "151654": {
94
- "content": "<|extra_8|>",
95
- "lstrip": false,
96
- "normalized": false,
97
- "rstrip": false,
98
- "single_word": false,
99
- "special": true
100
- },
101
- "151655": {
102
- "content": "<|extra_9|>",
103
- "lstrip": false,
104
- "normalized": false,
105
- "rstrip": false,
106
- "single_word": false,
107
- "special": true
108
- },
109
- "151656": {
110
- "content": "<|extra_10|>",
111
- "lstrip": false,
112
- "normalized": false,
113
- "rstrip": false,
114
- "single_word": false,
115
- "special": true
116
- },
117
- "151657": {
118
- "content": "<|extra_11|>",
119
- "lstrip": false,
120
- "normalized": false,
121
- "rstrip": false,
122
- "single_word": false,
123
- "special": true
124
- },
125
- "151658": {
126
- "content": "<|extra_12|>",
127
- "lstrip": false,
128
- "normalized": false,
129
- "rstrip": false,
130
- "single_word": false,
131
- "special": true
132
- },
133
- "151659": {
134
- "content": "<|extra_13|>",
135
- "lstrip": false,
136
- "normalized": false,
137
- "rstrip": false,
138
- "single_word": false,
139
- "special": true
140
- },
141
- "151660": {
142
- "content": "<|extra_14|>",
143
- "lstrip": false,
144
- "normalized": false,
145
- "rstrip": false,
146
- "single_word": false,
147
- "special": true
148
- },
149
- "151661": {
150
- "content": "<|extra_15|>",
151
- "lstrip": false,
152
- "normalized": false,
153
- "rstrip": false,
154
- "single_word": false,
155
- "special": true
156
- },
157
- "151662": {
158
- "content": "<|extra_16|>",
159
- "lstrip": false,
160
- "normalized": false,
161
- "rstrip": false,
162
- "single_word": false,
163
- "special": true
164
- },
165
- "151663": {
166
- "content": "<|extra_17|>",
167
- "lstrip": false,
168
- "normalized": false,
169
- "rstrip": false,
170
- "single_word": false,
171
- "special": true
172
- },
173
- "151664": {
174
- "content": "<|extra_18|>",
175
- "lstrip": false,
176
- "normalized": false,
177
- "rstrip": false,
178
- "single_word": false,
179
- "special": true
180
- },
181
- "151665": {
182
- "content": "<|extra_19|>",
183
- "lstrip": false,
184
- "normalized": false,
185
- "rstrip": false,
186
- "single_word": false,
187
- "special": true
188
- },
189
- "151666": {
190
- "content": "<|extra_20|>",
191
- "lstrip": false,
192
- "normalized": false,
193
- "rstrip": false,
194
- "single_word": false,
195
- "special": true
196
- },
197
- "151667": {
198
- "content": "<|extra_21|>",
199
- "lstrip": false,
200
- "normalized": false,
201
- "rstrip": false,
202
- "single_word": false,
203
- "special": true
204
- },
205
- "151668": {
206
- "content": "<|extra_22|>",
207
- "lstrip": false,
208
- "normalized": false,
209
- "rstrip": false,
210
- "single_word": false,
211
- "special": true
212
- },
213
- "151669": {
214
- "content": "<|extra_23|>",
215
- "lstrip": false,
216
- "normalized": false,
217
- "rstrip": false,
218
- "single_word": false,
219
- "special": true
220
- },
221
- "151670": {
222
- "content": "<|extra_24|>",
223
- "lstrip": false,
224
- "normalized": false,
225
- "rstrip": false,
226
- "single_word": false,
227
- "special": true
228
- },
229
- "151671": {
230
- "content": "<|extra_25|>",
231
- "lstrip": false,
232
- "normalized": false,
233
- "rstrip": false,
234
- "single_word": false,
235
- "special": true
236
- },
237
- "151672": {
238
- "content": "<|extra_26|>",
239
- "lstrip": false,
240
- "normalized": false,
241
- "rstrip": false,
242
- "single_word": false,
243
- "special": true
244
- },
245
- "151673": {
246
- "content": "<|extra_27|>",
247
- "lstrip": false,
248
- "normalized": false,
249
- "rstrip": false,
250
- "single_word": false,
251
- "special": true
252
- },
253
- "151674": {
254
- "content": "<|extra_28|>",
255
- "lstrip": false,
256
- "normalized": false,
257
- "rstrip": false,
258
- "single_word": false,
259
- "special": true
260
- },
261
- "151675": {
262
- "content": "<|extra_29|>",
263
- "lstrip": false,
264
- "normalized": false,
265
- "rstrip": false,
266
- "single_word": false,
267
- "special": true
268
- },
269
- "151676": {
270
- "content": "<|extra_30|>",
271
- "lstrip": false,
272
- "normalized": false,
273
- "rstrip": false,
274
- "single_word": false,
275
- "special": true
276
- },
277
- "151677": {
278
- "content": "<|extra_31|>",
279
- "lstrip": false,
280
- "normalized": false,
281
- "rstrip": false,
282
- "single_word": false,
283
- "special": true
284
- },
285
- "151678": {
286
- "content": "<|extra_32|>",
287
- "lstrip": false,
288
- "normalized": false,
289
- "rstrip": false,
290
- "single_word": false,
291
- "special": true
292
- },
293
- "151679": {
294
- "content": "<|extra_33|>",
295
- "lstrip": false,
296
- "normalized": false,
297
- "rstrip": false,
298
- "single_word": false,
299
- "special": true
300
- },
301
- "151680": {
302
- "content": "<|extra_34|>",
303
- "lstrip": false,
304
- "normalized": false,
305
- "rstrip": false,
306
- "single_word": false,
307
- "special": true
308
- },
309
- "151681": {
310
- "content": "<|extra_35|>",
311
- "lstrip": false,
312
- "normalized": false,
313
- "rstrip": false,
314
- "single_word": false,
315
- "special": true
316
- },
317
- "151682": {
318
- "content": "<|extra_36|>",
319
- "lstrip": false,
320
- "normalized": false,
321
- "rstrip": false,
322
- "single_word": false,
323
- "special": true
324
- },
325
- "151683": {
326
- "content": "<|extra_37|>",
327
- "lstrip": false,
328
- "normalized": false,
329
- "rstrip": false,
330
- "single_word": false,
331
- "special": true
332
- },
333
- "151684": {
334
- "content": "<|extra_38|>",
335
- "lstrip": false,
336
- "normalized": false,
337
- "rstrip": false,
338
- "single_word": false,
339
- "special": true
340
- },
341
- "151685": {
342
- "content": "<|extra_39|>",
343
- "lstrip": false,
344
- "normalized": false,
345
- "rstrip": false,
346
- "single_word": false,
347
- "special": true
348
- },
349
- "151686": {
350
- "content": "<|extra_40|>",
351
- "lstrip": false,
352
- "normalized": false,
353
- "rstrip": false,
354
- "single_word": false,
355
- "special": true
356
- },
357
- "151687": {
358
- "content": "<|extra_41|>",
359
- "lstrip": false,
360
- "normalized": false,
361
- "rstrip": false,
362
- "single_word": false,
363
- "special": true
364
- },
365
- "151688": {
366
- "content": "<|extra_42|>",
367
- "lstrip": false,
368
- "normalized": false,
369
- "rstrip": false,
370
- "single_word": false,
371
- "special": true
372
- },
373
- "151689": {
374
- "content": "<|extra_43|>",
375
- "lstrip": false,
376
- "normalized": false,
377
- "rstrip": false,
378
- "single_word": false,
379
- "special": true
380
- },
381
- "151690": {
382
- "content": "<|extra_44|>",
383
- "lstrip": false,
384
- "normalized": false,
385
- "rstrip": false,
386
- "single_word": false,
387
- "special": true
388
- },
389
- "151691": {
390
- "content": "<|extra_45|>",
391
- "lstrip": false,
392
- "normalized": false,
393
- "rstrip": false,
394
- "single_word": false,
395
- "special": true
396
- },
397
- "151692": {
398
- "content": "<|extra_46|>",
399
- "lstrip": false,
400
- "normalized": false,
401
- "rstrip": false,
402
- "single_word": false,
403
- "special": true
404
- },
405
- "151693": {
406
- "content": "<|extra_47|>",
407
- "lstrip": false,
408
- "normalized": false,
409
- "rstrip": false,
410
- "single_word": false,
411
- "special": true
412
- },
413
- "151694": {
414
- "content": "<|extra_48|>",
415
- "lstrip": false,
416
- "normalized": false,
417
- "rstrip": false,
418
- "single_word": false,
419
- "special": true
420
- },
421
- "151695": {
422
- "content": "<|extra_49|>",
423
- "lstrip": false,
424
- "normalized": false,
425
- "rstrip": false,
426
- "single_word": false,
427
- "special": true
428
- },
429
- "151696": {
430
- "content": "<|extra_50|>",
431
- "lstrip": false,
432
- "normalized": false,
433
- "rstrip": false,
434
- "single_word": false,
435
- "special": true
436
- },
437
- "151697": {
438
- "content": "<|extra_51|>",
439
- "lstrip": false,
440
- "normalized": false,
441
- "rstrip": false,
442
- "single_word": false,
443
- "special": true
444
- },
445
- "151698": {
446
- "content": "<|extra_52|>",
447
- "lstrip": false,
448
- "normalized": false,
449
- "rstrip": false,
450
- "single_word": false,
451
- "special": true
452
- },
453
- "151699": {
454
- "content": "<|extra_53|>",
455
- "lstrip": false,
456
- "normalized": false,
457
- "rstrip": false,
458
- "single_word": false,
459
- "special": true
460
- },
461
- "151700": {
462
- "content": "<|extra_54|>",
463
- "lstrip": false,
464
- "normalized": false,
465
- "rstrip": false,
466
- "single_word": false,
467
- "special": true
468
- },
469
- "151701": {
470
- "content": "<|extra_55|>",
471
- "lstrip": false,
472
- "normalized": false,
473
- "rstrip": false,
474
- "single_word": false,
475
- "special": true
476
- },
477
- "151702": {
478
- "content": "<|extra_56|>",
479
- "lstrip": false,
480
- "normalized": false,
481
- "rstrip": false,
482
- "single_word": false,
483
- "special": true
484
- },
485
- "151703": {
486
- "content": "<|extra_57|>",
487
- "lstrip": false,
488
- "normalized": false,
489
- "rstrip": false,
490
- "single_word": false,
491
- "special": true
492
- },
493
- "151704": {
494
- "content": "<|extra_58|>",
495
- "lstrip": false,
496
- "normalized": false,
497
- "rstrip": false,
498
- "single_word": false,
499
- "special": true
500
- },
501
- "151705": {
502
- "content": "<|extra_59|>",
503
- "lstrip": false,
504
- "normalized": false,
505
- "rstrip": false,
506
- "single_word": false,
507
- "special": true
508
- },
509
- "151706": {
510
- "content": "<|extra_60|>",
511
- "lstrip": false,
512
- "normalized": false,
513
- "rstrip": false,
514
- "single_word": false,
515
- "special": true
516
- },
517
- "151707": {
518
- "content": "<|extra_61|>",
519
- "lstrip": false,
520
- "normalized": false,
521
- "rstrip": false,
522
- "single_word": false,
523
- "special": true
524
- },
525
- "151708": {
526
- "content": "<|extra_62|>",
527
- "lstrip": false,
528
- "normalized": false,
529
- "rstrip": false,
530
- "single_word": false,
531
- "special": true
532
- },
533
- "151709": {
534
- "content": "<|extra_63|>",
535
- "lstrip": false,
536
- "normalized": false,
537
- "rstrip": false,
538
- "single_word": false,
539
- "special": true
540
- },
541
- "151710": {
542
- "content": "<|extra_64|>",
543
- "lstrip": false,
544
- "normalized": false,
545
- "rstrip": false,
546
- "single_word": false,
547
- "special": true
548
- },
549
- "151711": {
550
- "content": "<|extra_65|>",
551
- "lstrip": false,
552
- "normalized": false,
553
- "rstrip": false,
554
- "single_word": false,
555
- "special": true
556
- },
557
- "151712": {
558
- "content": "<|extra_66|>",
559
- "lstrip": false,
560
- "normalized": false,
561
- "rstrip": false,
562
- "single_word": false,
563
- "special": true
564
- },
565
- "151713": {
566
- "content": "<|extra_67|>",
567
- "lstrip": false,
568
- "normalized": false,
569
- "rstrip": false,
570
- "single_word": false,
571
- "special": true
572
- },
573
- "151714": {
574
- "content": "<|extra_68|>",
575
- "lstrip": false,
576
- "normalized": false,
577
- "rstrip": false,
578
- "single_word": false,
579
- "special": true
580
- },
581
- "151715": {
582
- "content": "<|extra_69|>",
583
- "lstrip": false,
584
- "normalized": false,
585
- "rstrip": false,
586
- "single_word": false,
587
- "special": true
588
- },
589
- "151716": {
590
- "content": "<|extra_70|>",
591
- "lstrip": false,
592
- "normalized": false,
593
- "rstrip": false,
594
- "single_word": false,
595
- "special": true
596
- },
597
- "151717": {
598
- "content": "<|extra_71|>",
599
- "lstrip": false,
600
- "normalized": false,
601
- "rstrip": false,
602
- "single_word": false,
603
- "special": true
604
- },
605
- "151718": {
606
- "content": "<|extra_72|>",
607
- "lstrip": false,
608
- "normalized": false,
609
- "rstrip": false,
610
- "single_word": false,
611
- "special": true
612
- },
613
- "151719": {
614
- "content": "<|extra_73|>",
615
- "lstrip": false,
616
- "normalized": false,
617
- "rstrip": false,
618
- "single_word": false,
619
- "special": true
620
- },
621
- "151720": {
622
- "content": "<|extra_74|>",
623
- "lstrip": false,
624
- "normalized": false,
625
- "rstrip": false,
626
- "single_word": false,
627
- "special": true
628
- },
629
- "151721": {
630
- "content": "<|extra_75|>",
631
- "lstrip": false,
632
- "normalized": false,
633
- "rstrip": false,
634
- "single_word": false,
635
- "special": true
636
- },
637
- "151722": {
638
- "content": "<|extra_76|>",
639
- "lstrip": false,
640
- "normalized": false,
641
- "rstrip": false,
642
- "single_word": false,
643
- "special": true
644
- },
645
- "151723": {
646
- "content": "<|extra_77|>",
647
- "lstrip": false,
648
- "normalized": false,
649
- "rstrip": false,
650
- "single_word": false,
651
- "special": true
652
- },
653
- "151724": {
654
- "content": "<|extra_78|>",
655
- "lstrip": false,
656
- "normalized": false,
657
- "rstrip": false,
658
- "single_word": false,
659
- "special": true
660
- },
661
- "151725": {
662
- "content": "<|extra_79|>",
663
- "lstrip": false,
664
- "normalized": false,
665
- "rstrip": false,
666
- "single_word": false,
667
- "special": true
668
- },
669
- "151726": {
670
- "content": "<|extra_80|>",
671
- "lstrip": false,
672
- "normalized": false,
673
- "rstrip": false,
674
- "single_word": false,
675
- "special": true
676
- },
677
- "151727": {
678
- "content": "<|extra_81|>",
679
- "lstrip": false,
680
- "normalized": false,
681
- "rstrip": false,
682
- "single_word": false,
683
- "special": true
684
- },
685
- "151728": {
686
- "content": "<|extra_82|>",
687
- "lstrip": false,
688
- "normalized": false,
689
- "rstrip": false,
690
- "single_word": false,
691
- "special": true
692
- },
693
- "151729": {
694
- "content": "<|extra_83|>",
695
- "lstrip": false,
696
- "normalized": false,
697
- "rstrip": false,
698
- "single_word": false,
699
- "special": true
700
- },
701
- "151730": {
702
- "content": "<|extra_84|>",
703
- "lstrip": false,
704
- "normalized": false,
705
- "rstrip": false,
706
- "single_word": false,
707
- "special": true
708
- },
709
- "151731": {
710
- "content": "<|extra_85|>",
711
- "lstrip": false,
712
- "normalized": false,
713
- "rstrip": false,
714
- "single_word": false,
715
- "special": true
716
- },
717
- "151732": {
718
- "content": "<|extra_86|>",
719
- "lstrip": false,
720
- "normalized": false,
721
- "rstrip": false,
722
- "single_word": false,
723
- "special": true
724
- },
725
- "151733": {
726
- "content": "<|extra_87|>",
727
- "lstrip": false,
728
- "normalized": false,
729
- "rstrip": false,
730
- "single_word": false,
731
- "special": true
732
- },
733
- "151734": {
734
- "content": "<|extra_88|>",
735
- "lstrip": false,
736
- "normalized": false,
737
- "rstrip": false,
738
- "single_word": false,
739
- "special": true
740
- },
741
- "151735": {
742
- "content": "<|extra_89|>",
743
- "lstrip": false,
744
- "normalized": false,
745
- "rstrip": false,
746
- "single_word": false,
747
- "special": true
748
- },
749
- "151736": {
750
- "content": "<|extra_90|>",
751
- "lstrip": false,
752
- "normalized": false,
753
- "rstrip": false,
754
- "single_word": false,
755
- "special": true
756
- },
757
- "151737": {
758
- "content": "<|extra_91|>",
759
- "lstrip": false,
760
- "normalized": false,
761
- "rstrip": false,
762
- "single_word": false,
763
- "special": true
764
- },
765
- "151738": {
766
- "content": "<|extra_92|>",
767
- "lstrip": false,
768
- "normalized": false,
769
- "rstrip": false,
770
- "single_word": false,
771
- "special": true
772
- },
773
- "151739": {
774
- "content": "<|extra_93|>",
775
- "lstrip": false,
776
- "normalized": false,
777
- "rstrip": false,
778
- "single_word": false,
779
- "special": true
780
- },
781
- "151740": {
782
- "content": "<|extra_94|>",
783
- "lstrip": false,
784
- "normalized": false,
785
- "rstrip": false,
786
- "single_word": false,
787
- "special": true
788
- },
789
- "151741": {
790
- "content": "<|extra_95|>",
791
- "lstrip": false,
792
- "normalized": false,
793
- "rstrip": false,
794
- "single_word": false,
795
- "special": true
796
- },
797
- "151742": {
798
- "content": "<|extra_96|>",
799
- "lstrip": false,
800
- "normalized": false,
801
- "rstrip": false,
802
- "single_word": false,
803
- "special": true
804
- },
805
- "151743": {
806
- "content": "<|extra_97|>",
807
- "lstrip": false,
808
- "normalized": false,
809
- "rstrip": false,
810
- "single_word": false,
811
- "special": true
812
- },
813
- "151744": {
814
- "content": "<|extra_98|>",
815
- "lstrip": false,
816
- "normalized": false,
817
- "rstrip": false,
818
- "single_word": false,
819
- "special": true
820
- },
821
- "151745": {
822
- "content": "<|extra_99|>",
823
- "lstrip": false,
824
- "normalized": false,
825
- "rstrip": false,
826
- "single_word": false,
827
- "special": true
828
- },
829
- "151746": {
830
- "content": "<|extra_100|>",
831
- "lstrip": false,
832
- "normalized": false,
833
- "rstrip": false,
834
- "single_word": false,
835
- "special": true
836
- },
837
- "151747": {
838
- "content": "<|extra_101|>",
839
- "lstrip": false,
840
- "normalized": false,
841
- "rstrip": false,
842
- "single_word": false,
843
- "special": true
844
- },
845
- "151748": {
846
- "content": "<|extra_102|>",
847
- "lstrip": false,
848
- "normalized": false,
849
- "rstrip": false,
850
- "single_word": false,
851
- "special": true
852
- },
853
- "151749": {
854
- "content": "<|extra_103|>",
855
- "lstrip": false,
856
- "normalized": false,
857
- "rstrip": false,
858
- "single_word": false,
859
- "special": true
860
- },
861
- "151750": {
862
- "content": "<|extra_104|>",
863
- "lstrip": false,
864
- "normalized": false,
865
- "rstrip": false,
866
- "single_word": false,
867
- "special": true
868
- },
869
- "151751": {
870
- "content": "<|extra_105|>",
871
- "lstrip": false,
872
- "normalized": false,
873
- "rstrip": false,
874
- "single_word": false,
875
- "special": true
876
- },
877
- "151752": {
878
- "content": "<|extra_106|>",
879
- "lstrip": false,
880
- "normalized": false,
881
- "rstrip": false,
882
- "single_word": false,
883
- "special": true
884
- },
885
- "151753": {
886
- "content": "<|extra_107|>",
887
- "lstrip": false,
888
- "normalized": false,
889
- "rstrip": false,
890
- "single_word": false,
891
- "special": true
892
- },
893
- "151754": {
894
- "content": "<|extra_108|>",
895
- "lstrip": false,
896
- "normalized": false,
897
- "rstrip": false,
898
- "single_word": false,
899
- "special": true
900
- },
901
- "151755": {
902
- "content": "<|extra_109|>",
903
- "lstrip": false,
904
- "normalized": false,
905
- "rstrip": false,
906
- "single_word": false,
907
- "special": true
908
- },
909
- "151756": {
910
- "content": "<|extra_110|>",
911
- "lstrip": false,
912
- "normalized": false,
913
- "rstrip": false,
914
- "single_word": false,
915
- "special": true
916
- },
917
- "151757": {
918
- "content": "<|extra_111|>",
919
- "lstrip": false,
920
- "normalized": false,
921
- "rstrip": false,
922
- "single_word": false,
923
- "special": true
924
- },
925
- "151758": {
926
- "content": "<|extra_112|>",
927
- "lstrip": false,
928
- "normalized": false,
929
- "rstrip": false,
930
- "single_word": false,
931
- "special": true
932
- },
933
- "151759": {
934
- "content": "<|extra_113|>",
935
- "lstrip": false,
936
- "normalized": false,
937
- "rstrip": false,
938
- "single_word": false,
939
- "special": true
940
- },
941
- "151760": {
942
- "content": "<|extra_114|>",
943
- "lstrip": false,
944
- "normalized": false,
945
- "rstrip": false,
946
- "single_word": false,
947
- "special": true
948
- },
949
- "151761": {
950
- "content": "<|extra_115|>",
951
- "lstrip": false,
952
- "normalized": false,
953
- "rstrip": false,
954
- "single_word": false,
955
- "special": true
956
- },
957
- "151762": {
958
- "content": "<|extra_116|>",
959
- "lstrip": false,
960
- "normalized": false,
961
- "rstrip": false,
962
- "single_word": false,
963
- "special": true
964
- },
965
- "151763": {
966
- "content": "<|extra_117|>",
967
- "lstrip": false,
968
- "normalized": false,
969
- "rstrip": false,
970
- "single_word": false,
971
- "special": true
972
- },
973
- "151764": {
974
- "content": "<|extra_118|>",
975
- "lstrip": false,
976
- "normalized": false,
977
- "rstrip": false,
978
- "single_word": false,
979
- "special": true
980
- },
981
- "151765": {
982
- "content": "<|extra_119|>",
983
- "lstrip": false,
984
- "normalized": false,
985
- "rstrip": false,
986
- "single_word": false,
987
- "special": true
988
- },
989
- "151766": {
990
- "content": "<|extra_120|>",
991
- "lstrip": false,
992
- "normalized": false,
993
- "rstrip": false,
994
- "single_word": false,
995
- "special": true
996
- },
997
- "151767": {
998
- "content": "<|extra_121|>",
999
- "lstrip": false,
1000
- "normalized": false,
1001
- "rstrip": false,
1002
- "single_word": false,
1003
- "special": true
1004
- },
1005
- "151768": {
1006
- "content": "<|extra_122|>",
1007
- "lstrip": false,
1008
- "normalized": false,
1009
- "rstrip": false,
1010
- "single_word": false,
1011
- "special": true
1012
- },
1013
- "151769": {
1014
- "content": "<|extra_123|>",
1015
- "lstrip": false,
1016
- "normalized": false,
1017
- "rstrip": false,
1018
- "single_word": false,
1019
- "special": true
1020
- },
1021
- "151770": {
1022
- "content": "<|extra_124|>",
1023
- "lstrip": false,
1024
- "normalized": false,
1025
- "rstrip": false,
1026
- "single_word": false,
1027
- "special": true
1028
- },
1029
- "151771": {
1030
- "content": "<|extra_125|>",
1031
- "lstrip": false,
1032
- "normalized": false,
1033
- "rstrip": false,
1034
- "single_word": false,
1035
- "special": true
1036
- },
1037
- "151772": {
1038
- "content": "<|extra_126|>",
1039
- "lstrip": false,
1040
- "normalized": false,
1041
- "rstrip": false,
1042
- "single_word": false,
1043
- "special": true
1044
- },
1045
- "151773": {
1046
- "content": "<|extra_127|>",
1047
- "lstrip": false,
1048
- "normalized": false,
1049
- "rstrip": false,
1050
- "single_word": false,
1051
- "special": true
1052
- },
1053
- "151774": {
1054
- "content": "<|extra_128|>",
1055
- "lstrip": false,
1056
- "normalized": false,
1057
- "rstrip": false,
1058
- "single_word": false,
1059
- "special": true
1060
- },
1061
- "151775": {
1062
- "content": "<|extra_129|>",
1063
- "lstrip": false,
1064
- "normalized": false,
1065
- "rstrip": false,
1066
- "single_word": false,
1067
- "special": true
1068
- },
1069
- "151776": {
1070
- "content": "<|extra_130|>",
1071
- "lstrip": false,
1072
- "normalized": false,
1073
- "rstrip": false,
1074
- "single_word": false,
1075
- "special": true
1076
- },
1077
- "151777": {
1078
- "content": "<|extra_131|>",
1079
- "lstrip": false,
1080
- "normalized": false,
1081
- "rstrip": false,
1082
- "single_word": false,
1083
- "special": true
1084
- },
1085
- "151778": {
1086
- "content": "<|extra_132|>",
1087
- "lstrip": false,
1088
- "normalized": false,
1089
- "rstrip": false,
1090
- "single_word": false,
1091
- "special": true
1092
- },
1093
- "151779": {
1094
- "content": "<|extra_133|>",
1095
- "lstrip": false,
1096
- "normalized": false,
1097
- "rstrip": false,
1098
- "single_word": false,
1099
- "special": true
1100
- },
1101
- "151780": {
1102
- "content": "<|extra_134|>",
1103
- "lstrip": false,
1104
- "normalized": false,
1105
- "rstrip": false,
1106
- "single_word": false,
1107
- "special": true
1108
- },
1109
- "151781": {
1110
- "content": "<|extra_135|>",
1111
- "lstrip": false,
1112
- "normalized": false,
1113
- "rstrip": false,
1114
- "single_word": false,
1115
- "special": true
1116
- },
1117
- "151782": {
1118
- "content": "<|extra_136|>",
1119
- "lstrip": false,
1120
- "normalized": false,
1121
- "rstrip": false,
1122
- "single_word": false,
1123
- "special": true
1124
- },
1125
- "151783": {
1126
- "content": "<|extra_137|>",
1127
- "lstrip": false,
1128
- "normalized": false,
1129
- "rstrip": false,
1130
- "single_word": false,
1131
- "special": true
1132
- },
1133
- "151784": {
1134
- "content": "<|extra_138|>",
1135
- "lstrip": false,
1136
- "normalized": false,
1137
- "rstrip": false,
1138
- "single_word": false,
1139
- "special": true
1140
- },
1141
- "151785": {
1142
- "content": "<|extra_139|>",
1143
- "lstrip": false,
1144
- "normalized": false,
1145
- "rstrip": false,
1146
- "single_word": false,
1147
- "special": true
1148
- },
1149
- "151786": {
1150
- "content": "<|extra_140|>",
1151
- "lstrip": false,
1152
- "normalized": false,
1153
- "rstrip": false,
1154
- "single_word": false,
1155
- "special": true
1156
- },
1157
- "151787": {
1158
- "content": "<|extra_141|>",
1159
- "lstrip": false,
1160
- "normalized": false,
1161
- "rstrip": false,
1162
- "single_word": false,
1163
- "special": true
1164
- },
1165
- "151788": {
1166
- "content": "<|extra_142|>",
1167
- "lstrip": false,
1168
- "normalized": false,
1169
- "rstrip": false,
1170
- "single_word": false,
1171
- "special": true
1172
- },
1173
- "151789": {
1174
- "content": "<|extra_143|>",
1175
- "lstrip": false,
1176
- "normalized": false,
1177
- "rstrip": false,
1178
- "single_word": false,
1179
- "special": true
1180
- },
1181
- "151790": {
1182
- "content": "<|extra_144|>",
1183
- "lstrip": false,
1184
- "normalized": false,
1185
- "rstrip": false,
1186
- "single_word": false,
1187
- "special": true
1188
- },
1189
- "151791": {
1190
- "content": "<|extra_145|>",
1191
- "lstrip": false,
1192
- "normalized": false,
1193
- "rstrip": false,
1194
- "single_word": false,
1195
- "special": true
1196
- },
1197
- "151792": {
1198
- "content": "<|extra_146|>",
1199
- "lstrip": false,
1200
- "normalized": false,
1201
- "rstrip": false,
1202
- "single_word": false,
1203
- "special": true
1204
- },
1205
- "151793": {
1206
- "content": "<|extra_147|>",
1207
- "lstrip": false,
1208
- "normalized": false,
1209
- "rstrip": false,
1210
- "single_word": false,
1211
- "special": true
1212
- },
1213
- "151794": {
1214
- "content": "<|extra_148|>",
1215
- "lstrip": false,
1216
- "normalized": false,
1217
- "rstrip": false,
1218
- "single_word": false,
1219
- "special": true
1220
- },
1221
- "151795": {
1222
- "content": "<|extra_149|>",
1223
- "lstrip": false,
1224
- "normalized": false,
1225
- "rstrip": false,
1226
- "single_word": false,
1227
- "special": true
1228
- },
1229
- "151796": {
1230
- "content": "<|extra_150|>",
1231
- "lstrip": false,
1232
- "normalized": false,
1233
- "rstrip": false,
1234
- "single_word": false,
1235
- "special": true
1236
- },
1237
- "151797": {
1238
- "content": "<|extra_151|>",
1239
- "lstrip": false,
1240
- "normalized": false,
1241
- "rstrip": false,
1242
- "single_word": false,
1243
- "special": true
1244
- },
1245
- "151798": {
1246
- "content": "<|extra_152|>",
1247
- "lstrip": false,
1248
- "normalized": false,
1249
- "rstrip": false,
1250
- "single_word": false,
1251
- "special": true
1252
- },
1253
- "151799": {
1254
- "content": "<|extra_153|>",
1255
- "lstrip": false,
1256
- "normalized": false,
1257
- "rstrip": false,
1258
- "single_word": false,
1259
- "special": true
1260
- },
1261
- "151800": {
1262
- "content": "<|extra_154|>",
1263
- "lstrip": false,
1264
- "normalized": false,
1265
- "rstrip": false,
1266
- "single_word": false,
1267
- "special": true
1268
- },
1269
- "151801": {
1270
- "content": "<|extra_155|>",
1271
- "lstrip": false,
1272
- "normalized": false,
1273
- "rstrip": false,
1274
- "single_word": false,
1275
- "special": true
1276
- },
1277
- "151802": {
1278
- "content": "<|extra_156|>",
1279
- "lstrip": false,
1280
- "normalized": false,
1281
- "rstrip": false,
1282
- "single_word": false,
1283
- "special": true
1284
- },
1285
- "151803": {
1286
- "content": "<|extra_157|>",
1287
- "lstrip": false,
1288
- "normalized": false,
1289
- "rstrip": false,
1290
- "single_word": false,
1291
- "special": true
1292
- },
1293
- "151804": {
1294
- "content": "<|extra_158|>",
1295
- "lstrip": false,
1296
- "normalized": false,
1297
- "rstrip": false,
1298
- "single_word": false,
1299
- "special": true
1300
- },
1301
- "151805": {
1302
- "content": "<|extra_159|>",
1303
- "lstrip": false,
1304
- "normalized": false,
1305
- "rstrip": false,
1306
- "single_word": false,
1307
- "special": true
1308
- },
1309
- "151806": {
1310
- "content": "<|extra_160|>",
1311
- "lstrip": false,
1312
- "normalized": false,
1313
- "rstrip": false,
1314
- "single_word": false,
1315
- "special": true
1316
- },
1317
- "151807": {
1318
- "content": "<|extra_161|>",
1319
- "lstrip": false,
1320
- "normalized": false,
1321
- "rstrip": false,
1322
- "single_word": false,
1323
- "special": true
1324
- },
1325
- "151808": {
1326
- "content": "<|extra_162|>",
1327
- "lstrip": false,
1328
- "normalized": false,
1329
- "rstrip": false,
1330
- "single_word": false,
1331
- "special": true
1332
- },
1333
- "151809": {
1334
- "content": "<|extra_163|>",
1335
- "lstrip": false,
1336
- "normalized": false,
1337
- "rstrip": false,
1338
- "single_word": false,
1339
- "special": true
1340
- },
1341
- "151810": {
1342
- "content": "<|extra_164|>",
1343
- "lstrip": false,
1344
- "normalized": false,
1345
- "rstrip": false,
1346
- "single_word": false,
1347
- "special": true
1348
- },
1349
- "151811": {
1350
- "content": "<|extra_165|>",
1351
- "lstrip": false,
1352
- "normalized": false,
1353
- "rstrip": false,
1354
- "single_word": false,
1355
- "special": true
1356
- },
1357
- "151812": {
1358
- "content": "<|extra_166|>",
1359
- "lstrip": false,
1360
- "normalized": false,
1361
- "rstrip": false,
1362
- "single_word": false,
1363
- "special": true
1364
- },
1365
- "151813": {
1366
- "content": "<|extra_167|>",
1367
- "lstrip": false,
1368
- "normalized": false,
1369
- "rstrip": false,
1370
- "single_word": false,
1371
- "special": true
1372
- },
1373
- "151814": {
1374
- "content": "<|extra_168|>",
1375
- "lstrip": false,
1376
- "normalized": false,
1377
- "rstrip": false,
1378
- "single_word": false,
1379
- "special": true
1380
- },
1381
- "151815": {
1382
- "content": "<|extra_169|>",
1383
- "lstrip": false,
1384
- "normalized": false,
1385
- "rstrip": false,
1386
- "single_word": false,
1387
- "special": true
1388
- },
1389
- "151816": {
1390
- "content": "<|extra_170|>",
1391
- "lstrip": false,
1392
- "normalized": false,
1393
- "rstrip": false,
1394
- "single_word": false,
1395
- "special": true
1396
- },
1397
- "151817": {
1398
- "content": "<|extra_171|>",
1399
- "lstrip": false,
1400
- "normalized": false,
1401
- "rstrip": false,
1402
- "single_word": false,
1403
- "special": true
1404
- },
1405
- "151818": {
1406
- "content": "<|extra_172|>",
1407
- "lstrip": false,
1408
- "normalized": false,
1409
- "rstrip": false,
1410
- "single_word": false,
1411
- "special": true
1412
- },
1413
- "151819": {
1414
- "content": "<|extra_173|>",
1415
- "lstrip": false,
1416
- "normalized": false,
1417
- "rstrip": false,
1418
- "single_word": false,
1419
- "special": true
1420
- },
1421
- "151820": {
1422
- "content": "<|extra_174|>",
1423
- "lstrip": false,
1424
- "normalized": false,
1425
- "rstrip": false,
1426
- "single_word": false,
1427
- "special": true
1428
- },
1429
- "151821": {
1430
- "content": "<|extra_175|>",
1431
- "lstrip": false,
1432
- "normalized": false,
1433
- "rstrip": false,
1434
- "single_word": false,
1435
- "special": true
1436
- },
1437
- "151822": {
1438
- "content": "<|extra_176|>",
1439
- "lstrip": false,
1440
- "normalized": false,
1441
- "rstrip": false,
1442
- "single_word": false,
1443
- "special": true
1444
- },
1445
- "151823": {
1446
- "content": "<|extra_177|>",
1447
- "lstrip": false,
1448
- "normalized": false,
1449
- "rstrip": false,
1450
- "single_word": false,
1451
- "special": true
1452
- },
1453
- "151824": {
1454
- "content": "<|extra_178|>",
1455
- "lstrip": false,
1456
- "normalized": false,
1457
- "rstrip": false,
1458
- "single_word": false,
1459
- "special": true
1460
- },
1461
- "151825": {
1462
- "content": "<|extra_179|>",
1463
- "lstrip": false,
1464
- "normalized": false,
1465
- "rstrip": false,
1466
- "single_word": false,
1467
- "special": true
1468
- },
1469
- "151826": {
1470
- "content": "<|extra_180|>",
1471
- "lstrip": false,
1472
- "normalized": false,
1473
- "rstrip": false,
1474
- "single_word": false,
1475
- "special": true
1476
- },
1477
- "151827": {
1478
- "content": "<|extra_181|>",
1479
- "lstrip": false,
1480
- "normalized": false,
1481
- "rstrip": false,
1482
- "single_word": false,
1483
- "special": true
1484
- },
1485
- "151828": {
1486
- "content": "<|extra_182|>",
1487
- "lstrip": false,
1488
- "normalized": false,
1489
- "rstrip": false,
1490
- "single_word": false,
1491
- "special": true
1492
- },
1493
- "151829": {
1494
- "content": "<|extra_183|>",
1495
- "lstrip": false,
1496
- "normalized": false,
1497
- "rstrip": false,
1498
- "single_word": false,
1499
- "special": true
1500
- },
1501
- "151830": {
1502
- "content": "<|extra_184|>",
1503
- "lstrip": false,
1504
- "normalized": false,
1505
- "rstrip": false,
1506
- "single_word": false,
1507
- "special": true
1508
- },
1509
- "151831": {
1510
- "content": "<|extra_185|>",
1511
- "lstrip": false,
1512
- "normalized": false,
1513
- "rstrip": false,
1514
- "single_word": false,
1515
- "special": true
1516
- },
1517
- "151832": {
1518
- "content": "<|extra_186|>",
1519
- "lstrip": false,
1520
- "normalized": false,
1521
- "rstrip": false,
1522
- "single_word": false,
1523
- "special": true
1524
- },
1525
- "151833": {
1526
- "content": "<|extra_187|>",
1527
- "lstrip": false,
1528
- "normalized": false,
1529
- "rstrip": false,
1530
- "single_word": false,
1531
- "special": true
1532
- },
1533
- "151834": {
1534
- "content": "<|extra_188|>",
1535
- "lstrip": false,
1536
- "normalized": false,
1537
- "rstrip": false,
1538
- "single_word": false,
1539
- "special": true
1540
- },
1541
- "151835": {
1542
- "content": "<|extra_189|>",
1543
- "lstrip": false,
1544
- "normalized": false,
1545
- "rstrip": false,
1546
- "single_word": false,
1547
- "special": true
1548
- },
1549
- "151836": {
1550
- "content": "<|extra_190|>",
1551
- "lstrip": false,
1552
- "normalized": false,
1553
- "rstrip": false,
1554
- "single_word": false,
1555
- "special": true
1556
- },
1557
- "151837": {
1558
- "content": "<|extra_191|>",
1559
- "lstrip": false,
1560
- "normalized": false,
1561
- "rstrip": false,
1562
- "single_word": false,
1563
- "special": true
1564
- },
1565
- "151838": {
1566
- "content": "<|extra_192|>",
1567
- "lstrip": false,
1568
- "normalized": false,
1569
- "rstrip": false,
1570
- "single_word": false,
1571
- "special": true
1572
- },
1573
- "151839": {
1574
- "content": "<|extra_193|>",
1575
- "lstrip": false,
1576
- "normalized": false,
1577
- "rstrip": false,
1578
- "single_word": false,
1579
- "special": true
1580
- },
1581
- "151840": {
1582
- "content": "<|extra_194|>",
1583
- "lstrip": false,
1584
- "normalized": false,
1585
- "rstrip": false,
1586
- "single_word": false,
1587
- "special": true
1588
- },
1589
- "151841": {
1590
- "content": "<|extra_195|>",
1591
- "lstrip": false,
1592
- "normalized": false,
1593
- "rstrip": false,
1594
- "single_word": false,
1595
- "special": true
1596
- },
1597
- "151842": {
1598
- "content": "<|extra_196|>",
1599
- "lstrip": false,
1600
- "normalized": false,
1601
- "rstrip": false,
1602
- "single_word": false,
1603
- "special": true
1604
- },
1605
- "151843": {
1606
- "content": "<|extra_197|>",
1607
- "lstrip": false,
1608
- "normalized": false,
1609
- "rstrip": false,
1610
- "single_word": false,
1611
- "special": true
1612
- },
1613
- "151844": {
1614
- "content": "<|extra_198|>",
1615
- "lstrip": false,
1616
- "normalized": false,
1617
- "rstrip": false,
1618
- "single_word": false,
1619
- "special": true
1620
- },
1621
- "151845": {
1622
- "content": "<|extra_199|>",
1623
- "lstrip": false,
1624
- "normalized": false,
1625
- "rstrip": false,
1626
- "single_word": false,
1627
- "special": true
1628
- },
1629
- "151846": {
1630
- "content": "<|extra_200|>",
1631
- "lstrip": false,
1632
- "normalized": false,
1633
- "rstrip": false,
1634
- "single_word": false,
1635
- "special": true
1636
- },
1637
- "151847": {
1638
- "content": "<|extra_201|>",
1639
- "lstrip": false,
1640
- "normalized": false,
1641
- "rstrip": false,
1642
- "single_word": false,
1643
- "special": true
1644
- },
1645
- "151848": {
1646
- "content": "<|extra_202|>",
1647
- "lstrip": false,
1648
- "normalized": false,
1649
- "rstrip": false,
1650
- "single_word": false,
1651
- "special": true
1652
- },
1653
- "151849": {
1654
- "content": "<|extra_203|>",
1655
- "lstrip": false,
1656
- "normalized": false,
1657
- "rstrip": false,
1658
- "single_word": false,
1659
- "special": true
1660
- },
1661
- "151850": {
1662
- "content": "<|extra_204|>",
1663
- "lstrip": false,
1664
- "normalized": false,
1665
- "rstrip": false,
1666
- "single_word": false,
1667
- "special": true
1668
- },
1669
- "151851": {
1670
- "content": "<ref>",
1671
- "lstrip": false,
1672
- "normalized": false,
1673
- "rstrip": false,
1674
- "single_word": true,
1675
- "special": true
1676
- },
1677
- "151852": {
1678
- "content": "</ref>",
1679
- "lstrip": false,
1680
- "normalized": false,
1681
- "rstrip": false,
1682
- "single_word": true,
1683
- "special": true
1684
- },
1685
- "151853": {
1686
- "content": "<box>",
1687
- "lstrip": false,
1688
- "normalized": false,
1689
- "rstrip": false,
1690
- "single_word": true,
1691
- "special": true
1692
- },
1693
- "151854": {
1694
- "content": "</box>",
1695
- "lstrip": false,
1696
- "normalized": false,
1697
- "rstrip": false,
1698
- "single_word": true,
1699
- "special": true
1700
- },
1701
- "151855": {
1702
- "content": "<quad>",
1703
- "lstrip": false,
1704
- "normalized": false,
1705
- "rstrip": false,
1706
- "single_word": true,
1707
- "special": true
1708
- },
1709
- "151856": {
1710
- "content": "</quad>",
1711
- "lstrip": false,
1712
- "normalized": false,
1713
- "rstrip": false,
1714
- "single_word": true,
1715
- "special": true
1716
- },
1717
- "151857": {
1718
- "content": "<img>",
1719
- "lstrip": false,
1720
- "normalized": false,
1721
- "rstrip": false,
1722
- "single_word": true,
1723
- "special": true
1724
- },
1725
- "151858": {
1726
- "content": "</img>",
1727
- "lstrip": false,
1728
- "normalized": false,
1729
- "rstrip": false,
1730
- "single_word": true,
1731
- "special": true
1732
- },
1733
- "151859": {
1734
- "content": "<imgpad>",
1735
- "lstrip": false,
1736
- "normalized": false,
1737
- "rstrip": false,
1738
- "single_word": true,
1739
- "special": true
1740
- }
1741
- },
1742
- "additional_special_tokens": [
1743
- "<|endoftext|>",
1744
- "<|im_start|>",
1745
- "<|im_end|>",
1746
- "<ref>",
1747
- "</ref>",
1748
- "<box>",
1749
- "</box>",
1750
- "<quad>",
1751
- "</quad>",
1752
- "<img>",
1753
- "</img>",
1754
- "<imgpad>"
1755
- ],
1756
- "message_start_token": "<|im_start|>",
1757
- "message_end_token": "<|im_end|>",
1758
- "img_start_token": "<img>",
1759
- "img_end_token": "</img>",
1760
- "img_pad_token": "<imgpad>",
1761
- "image_token": "<imgpad>",
1762
  "bos_token": null,
1763
- "clean_up_tokenization_spaces": false,
1764
- "eos_token": "<|im_end|>",
1765
- "errors": "replace",
1766
- "extra_special_tokens": {},
1767
- "model_max_length": 131072,
1768
  "pad_token": "<|endoftext|>",
1769
- "split_special_tokens": false,
1770
  "tokenizer_class": "Qwen2Tokenizer",
1771
- "unk_token": null
1772
  }
1773
-
1774
-
 
1
  {
2
+ "add_prefix_space": null,
3
+ "backend": "tokenizers",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4
  "bos_token": null,
5
+ "eos_token": "<|endoftext|>",
6
+ "model_max_length": 1000000000000000019884624838656,
 
 
 
7
  "pad_token": "<|endoftext|>",
 
8
  "tokenizer_class": "Qwen2Tokenizer",
9
+ "unk_token": "<|endoftext|>"
10
  }