Hugging Face's logo

PleIAs
/
Pleias-Pico

Safetensors
llama
Model card Files
xet
Community
Pclanglais commited on
Commit
e2c75d6
·
verified ·
1 Parent(s): 9fac57f

Upload folder using huggingface_hub

Browse files
Files changed (3) hide show
  1. special_tokens_map.json +7 -23
  2. tokenizer.json +44 -156
  3. tokenizer_config.json +17 -161
special_tokens_map.json CHANGED
@@ -1,30 +1,14 @@
1
  {
2
  "additional_special_tokens": [
3
- "<|tool_call_start|>",
4
- "<|tool_call_end|>",
5
- "<|tool_list_start|>",
6
- "<|tool_list_end|>",
7
- "<|source_analysis_start|>",
8
- "<|source_analysis_end|>",
9
  "<|source_start|>",
10
  "<|source_end|>",
11
- "<|im_start|>",
12
- "<|im_end|>",
 
 
13
  "<|answer_start|>",
14
- "<|answer_end|>",
15
- "<|text_start|>",
16
- "<|text_end|>",
17
- "<|translation_start|>",
18
- "<|translation_end|>",
19
- "<|back_translation_start|>",
20
- "<|back_translation_end|>",
21
- "<|ocr_correction_start|>",
22
- "<|ocr_correction_end|>",
23
- "<|json_scheme_start|>",
24
- "<|json_scheme_end|>",
25
- "<|source_interpretation_start|>",
26
- "<|source_interpretation_end|>",
27
- "<|query_start|>",
28
- "<|query_end|>"
29
  ]
30
  }
 
1
  {
2
  "additional_special_tokens": [
3
+ "<|query_start|>",
4
+ "<|query_end|>",
 
 
 
 
5
  "<|source_start|>",
6
  "<|source_end|>",
7
+ "<|source_id_start|>",
8
+ "<|source_id_end|>",
9
+ "<|source_analysis_start|>",
10
+ "<|source_analysis_end|>",
11
  "<|answer_start|>",
12
+ "<|answer_end|>"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
13
  ]
14
  }
tokenizer.json CHANGED
@@ -39,153 +39,9 @@
39
  "normalized": false,
40
  "special": true
41
  },
42
- {
43
- "id": 65510,
44
- "content": "<|tool_call_start|>",
45
- "single_word": false,
46
- "lstrip": false,
47
- "rstrip": false,
48
- "normalized": false,
49
- "special": true
50
- },
51
- {
52
- "id": 65511,
53
- "content": "<|tool_call_end|>",
54
- "single_word": false,
55
- "lstrip": false,
56
- "rstrip": false,
57
- "normalized": false,
58
- "special": true
59
- },
60
- {
61
- "id": 65512,
62
- "content": "<|tool_list_start|>",
63
- "single_word": false,
64
- "lstrip": false,
65
- "rstrip": false,
66
- "normalized": false,
67
- "special": true
68
- },
69
- {
70
- "id": 65513,
71
- "content": "<|tool_list_end|>",
72
- "single_word": false,
73
- "lstrip": false,
74
- "rstrip": false,
75
- "normalized": false,
76
- "special": true
77
- },
78
- {
79
- "id": 65514,
80
- "content": "<|source_analysis_start|>",
81
- "single_word": false,
82
- "lstrip": false,
83
- "rstrip": false,
84
- "normalized": false,
85
- "special": true
86
- },
87
- {
88
- "id": 65515,
89
- "content": "<|source_analysis_end|>",
90
- "single_word": false,
91
- "lstrip": false,
92
- "rstrip": false,
93
- "normalized": false,
94
- "special": true
95
- },
96
- {
97
- "id": 65516,
98
- "content": "<|source_start|>",
99
- "single_word": false,
100
- "lstrip": false,
101
- "rstrip": false,
102
- "normalized": false,
103
- "special": true
104
- },
105
- {
106
- "id": 65517,
107
- "content": "<|source_end|>",
108
- "single_word": false,
109
- "lstrip": false,
110
- "rstrip": false,
111
- "normalized": false,
112
- "special": true
113
- },
114
- {
115
- "id": 65518,
116
- "content": "<|im_start|>",
117
- "single_word": false,
118
- "lstrip": false,
119
- "rstrip": false,
120
- "normalized": false,
121
- "special": true
122
- },
123
- {
124
- "id": 65519,
125
- "content": "<|im_end|>",
126
- "single_word": false,
127
- "lstrip": false,
128
- "rstrip": false,
129
- "normalized": false,
130
- "special": true
131
- },
132
- {
133
- "id": 65520,
134
- "content": "<|answer_start|>",
135
- "single_word": false,
136
- "lstrip": false,
137
- "rstrip": false,
138
- "normalized": false,
139
- "special": true
140
- },
141
- {
142
- "id": 65521,
143
- "content": "<|answer_end|>",
144
- "single_word": false,
145
- "lstrip": false,
146
- "rstrip": false,
147
- "normalized": false,
148
- "special": true
149
- },
150
- {
151
- "id": 65522,
152
- "content": "<|text_start|>",
153
- "single_word": false,
154
- "lstrip": false,
155
- "rstrip": false,
156
- "normalized": false,
157
- "special": true
158
- },
159
- {
160
- "id": 65523,
161
- "content": "<|text_end|>",
162
- "single_word": false,
163
- "lstrip": false,
164
- "rstrip": false,
165
- "normalized": false,
166
- "special": true
167
- },
168
- {
169
- "id": 65524,
170
- "content": "<|translation_start|>",
171
- "single_word": false,
172
- "lstrip": false,
173
- "rstrip": false,
174
- "normalized": false,
175
- "special": true
176
- },
177
- {
178
- "id": 65525,
179
- "content": "<|translation_end|>",
180
- "single_word": false,
181
- "lstrip": false,
182
- "rstrip": false,
183
- "normalized": false,
184
- "special": true
185
- },
186
  {
187
  "id": 65526,
188
- "content": "<|back_translation_start|>",
189
  "single_word": false,
190
  "lstrip": false,
191
  "rstrip": false,
@@ -194,7 +50,7 @@
194
  },
195
  {
196
  "id": 65527,
197
- "content": "<|back_translation_end|>",
198
  "single_word": false,
199
  "lstrip": false,
200
  "rstrip": false,
@@ -203,7 +59,7 @@
203
  },
204
  {
205
  "id": 65528,
206
- "content": "<|ocr_correction_start|>",
207
  "single_word": false,
208
  "lstrip": false,
209
  "rstrip": false,
@@ -212,7 +68,7 @@
212
  },
213
  {
214
  "id": 65529,
215
- "content": "<|ocr_correction_end|>",
216
  "single_word": false,
217
  "lstrip": false,
218
  "rstrip": false,
@@ -221,7 +77,7 @@
221
  },
222
  {
223
  "id": 65530,
224
- "content": "<|json_scheme_start|>",
225
  "single_word": false,
226
  "lstrip": false,
227
  "rstrip": false,
@@ -230,7 +86,7 @@
230
  },
231
  {
232
  "id": 65531,
233
- "content": "<|json_scheme_end|>",
234
  "single_word": false,
235
  "lstrip": false,
236
  "rstrip": false,
@@ -239,7 +95,7 @@
239
  },
240
  {
241
  "id": 65532,
242
- "content": "<|source_interpretation_start|>",
243
  "single_word": false,
244
  "lstrip": false,
245
  "rstrip": false,
@@ -248,7 +104,7 @@
248
  },
249
  {
250
  "id": 65533,
251
- "content": "<|source_interpretation_end|>",
252
  "single_word": false,
253
  "lstrip": false,
254
  "rstrip": false,
@@ -257,7 +113,7 @@
257
  },
258
  {
259
  "id": 65534,
260
- "content": "<|query_start|>",
261
  "single_word": false,
262
  "lstrip": false,
263
  "rstrip": false,
@@ -266,7 +122,7 @@
266
  },
267
  {
268
  "id": 65535,
269
- "content": "<|query_end|>",
270
  "single_word": false,
271
  "lstrip": false,
272
  "rstrip": false,
@@ -65911,7 +65767,23 @@
65911
  "reste": 65506,
65912
  "ĠMonat": 65507,
65913
  "å·¥ä½ľ": 65508,
65914
- "Ġdémocratique": 65509
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
65915
  },
65916
  "merges": [
65917
  "Ġ Ġ",
@@ -131176,7 +131048,23 @@
131176
  "res te",
131177
  "ĠMon at",
131178
  "å·¥ ä½ľ",
131179
- "Ġdémocra tique"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
131180
  ]
131181
  }
131182
  }
 
39
  "normalized": false,
40
  "special": true
41
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
42
  {
43
  "id": 65526,
44
+ "content": "<|query_start|>",
45
  "single_word": false,
46
  "lstrip": false,
47
  "rstrip": false,
 
50
  },
51
  {
52
  "id": 65527,
53
+ "content": "<|query_end|>",
54
  "single_word": false,
55
  "lstrip": false,
56
  "rstrip": false,
 
59
  },
60
  {
61
  "id": 65528,
62
+ "content": "<|source_start|>",
63
  "single_word": false,
64
  "lstrip": false,
65
  "rstrip": false,
 
68
  },
69
  {
70
  "id": 65529,
71
+ "content": "<|source_end|>",
72
  "single_word": false,
73
  "lstrip": false,
74
  "rstrip": false,
 
77
  },
78
  {
79
  "id": 65530,
80
+ "content": "<|source_id_start|>",
81
  "single_word": false,
82
  "lstrip": false,
83
  "rstrip": false,
 
86
  },
87
  {
88
  "id": 65531,
89
+ "content": "<|source_id_end|>",
90
  "single_word": false,
91
  "lstrip": false,
92
  "rstrip": false,
 
95
  },
96
  {
97
  "id": 65532,
98
+ "content": "<|source_analysis_start|>",
99
  "single_word": false,
100
  "lstrip": false,
101
  "rstrip": false,
 
104
  },
105
  {
106
  "id": 65533,
107
+ "content": "<|source_analysis_end|>",
108
  "single_word": false,
109
  "lstrip": false,
110
  "rstrip": false,
 
113
  },
114
  {
115
  "id": 65534,
116
+ "content": "<|answer_start|>",
117
  "single_word": false,
118
  "lstrip": false,
119
  "rstrip": false,
 
122
  },
123
  {
124
  "id": 65535,
125
+ "content": "<|answer_end|>",
126
  "single_word": false,
127
  "lstrip": false,
128
  "rstrip": false,
 
65767
  "reste": 65506,
65768
  "ĠMonat": 65507,
65769
  "å·¥ä½ľ": 65508,
65770
+ "Ġdémocratique": 65509,
65771
+ "Ġsejam": 65510,
65772
+ "ÑĥбеÑĢ": 65511,
65773
+ "Ġaccessoires": 65512,
65774
+ "Ġ709": 65513,
65775
+ "Ġménages": 65514,
65776
+ "Ġprodukty": 65515,
65777
+ "Ġsuum": 65516,
65778
+ "Ġinfluenc": 65517,
65779
+ "Ġcatching": 65518,
65780
+ "ĠÏĮÏĦαν": 65519,
65781
+ "737": 65520,
65782
+ "ĠоÑĤде": 65521,
65783
+ "Ġë°ı": 65522,
65784
+ "symptoms": 65523,
65785
+ "apac": 65524,
65786
+ "Ġdépou": 65525
65787
  },
65788
  "merges": [
65789
  "Ġ Ġ",
 
131048
  "res te",
131049
  "ĠMon at",
131050
  "å·¥ ä½ľ",
131051
+ "Ġdémocra tique",
131052
+ "Ġse jam",
131053
+ "Ñĥб еÑĢ",
131054
+ "Ġaccess oires",
131055
+ "Ġ7 09",
131056
+ "Ġmén ages",
131057
+ "Ġproduk ty",
131058
+ "Ġsu um",
131059
+ "Ġinflu enc",
131060
+ "Ġcatch ing",
131061
+ "ĠÏĮ ÏĦαν",
131062
+ "7 37",
131063
+ "ĠоÑĤ де",
131064
+ "Ġë° ı",
131065
+ "sym ptoms",
131066
+ "ap ac",
131067
+ "Ġdép ou"
131068
  ]
131069
  }
131070
  }
tokenizer_config.json CHANGED
@@ -32,136 +32,8 @@
32
  "single_word": false,
33
  "special": true
34
  },
35
- "65510": {
36
- "content": "<|tool_call_start|>",
37
- "lstrip": false,
38
- "normalized": false,
39
- "rstrip": false,
40
- "single_word": false,
41
- "special": true
42
- },
43
- "65511": {
44
- "content": "<|tool_call_end|>",
45
- "lstrip": false,
46
- "normalized": false,
47
- "rstrip": false,
48
- "single_word": false,
49
- "special": true
50
- },
51
- "65512": {
52
- "content": "<|tool_list_start|>",
53
- "lstrip": false,
54
- "normalized": false,
55
- "rstrip": false,
56
- "single_word": false,
57
- "special": true
58
- },
59
- "65513": {
60
- "content": "<|tool_list_end|>",
61
- "lstrip": false,
62
- "normalized": false,
63
- "rstrip": false,
64
- "single_word": false,
65
- "special": true
66
- },
67
- "65514": {
68
- "content": "<|source_analysis_start|>",
69
- "lstrip": false,
70
- "normalized": false,
71
- "rstrip": false,
72
- "single_word": false,
73
- "special": true
74
- },
75
- "65515": {
76
- "content": "<|source_analysis_end|>",
77
- "lstrip": false,
78
- "normalized": false,
79
- "rstrip": false,
80
- "single_word": false,
81
- "special": true
82
- },
83
- "65516": {
84
- "content": "<|source_start|>",
85
- "lstrip": false,
86
- "normalized": false,
87
- "rstrip": false,
88
- "single_word": false,
89
- "special": true
90
- },
91
- "65517": {
92
- "content": "<|source_end|>",
93
- "lstrip": false,
94
- "normalized": false,
95
- "rstrip": false,
96
- "single_word": false,
97
- "special": true
98
- },
99
- "65518": {
100
- "content": "<|im_start|>",
101
- "lstrip": false,
102
- "normalized": false,
103
- "rstrip": false,
104
- "single_word": false,
105
- "special": true
106
- },
107
- "65519": {
108
- "content": "<|im_end|>",
109
- "lstrip": false,
110
- "normalized": false,
111
- "rstrip": false,
112
- "single_word": false,
113
- "special": true
114
- },
115
- "65520": {
116
- "content": "<|answer_start|>",
117
- "lstrip": false,
118
- "normalized": false,
119
- "rstrip": false,
120
- "single_word": false,
121
- "special": true
122
- },
123
- "65521": {
124
- "content": "<|answer_end|>",
125
- "lstrip": false,
126
- "normalized": false,
127
- "rstrip": false,
128
- "single_word": false,
129
- "special": true
130
- },
131
- "65522": {
132
- "content": "<|text_start|>",
133
- "lstrip": false,
134
- "normalized": false,
135
- "rstrip": false,
136
- "single_word": false,
137
- "special": true
138
- },
139
- "65523": {
140
- "content": "<|text_end|>",
141
- "lstrip": false,
142
- "normalized": false,
143
- "rstrip": false,
144
- "single_word": false,
145
- "special": true
146
- },
147
- "65524": {
148
- "content": "<|translation_start|>",
149
- "lstrip": false,
150
- "normalized": false,
151
- "rstrip": false,
152
- "single_word": false,
153
- "special": true
154
- },
155
- "65525": {
156
- "content": "<|translation_end|>",
157
- "lstrip": false,
158
- "normalized": false,
159
- "rstrip": false,
160
- "single_word": false,
161
- "special": true
162
- },
163
  "65526": {
164
- "content": "<|back_translation_start|>",
165
  "lstrip": false,
166
  "normalized": false,
167
  "rstrip": false,
@@ -169,7 +41,7 @@
169
  "special": true
170
  },
171
  "65527": {
172
- "content": "<|back_translation_end|>",
173
  "lstrip": false,
174
  "normalized": false,
175
  "rstrip": false,
@@ -177,7 +49,7 @@
177
  "special": true
178
  },
179
  "65528": {
180
- "content": "<|ocr_correction_start|>",
181
  "lstrip": false,
182
  "normalized": false,
183
  "rstrip": false,
@@ -185,7 +57,7 @@
185
  "special": true
186
  },
187
  "65529": {
188
- "content": "<|ocr_correction_end|>",
189
  "lstrip": false,
190
  "normalized": false,
191
  "rstrip": false,
@@ -193,7 +65,7 @@
193
  "special": true
194
  },
195
  "65530": {
196
- "content": "<|json_scheme_start|>",
197
  "lstrip": false,
198
  "normalized": false,
199
  "rstrip": false,
@@ -201,7 +73,7 @@
201
  "special": true
202
  },
203
  "65531": {
204
- "content": "<|json_scheme_end|>",
205
  "lstrip": false,
206
  "normalized": false,
207
  "rstrip": false,
@@ -209,7 +81,7 @@
209
  "special": true
210
  },
211
  "65532": {
212
- "content": "<|source_interpretation_start|>",
213
  "lstrip": false,
214
  "normalized": false,
215
  "rstrip": false,
@@ -217,7 +89,7 @@
217
  "special": true
218
  },
219
  "65533": {
220
- "content": "<|source_interpretation_end|>",
221
  "lstrip": false,
222
  "normalized": false,
223
  "rstrip": false,
@@ -225,7 +97,7 @@
225
  "special": true
226
  },
227
  "65534": {
228
- "content": "<|query_start|>",
229
  "lstrip": false,
230
  "normalized": false,
231
  "rstrip": false,
@@ -233,7 +105,7 @@
233
  "special": true
234
  },
235
  "65535": {
236
- "content": "<|query_end|>",
237
  "lstrip": false,
238
  "normalized": false,
239
  "rstrip": false,
@@ -242,32 +114,16 @@
242
  }
243
  },
244
  "additional_special_tokens": [
245
- "<|tool_call_start|>",
246
- "<|tool_call_end|>",
247
- "<|tool_list_start|>",
248
- "<|tool_list_end|>",
249
- "<|source_analysis_start|>",
250
- "<|source_analysis_end|>",
251
  "<|source_start|>",
252
  "<|source_end|>",
253
- "<|im_start|>",
254
- "<|im_end|>",
 
 
255
  "<|answer_start|>",
256
- "<|answer_end|>",
257
- "<|text_start|>",
258
- "<|text_end|>",
259
- "<|translation_start|>",
260
- "<|translation_end|>",
261
- "<|back_translation_start|>",
262
- "<|back_translation_end|>",
263
- "<|ocr_correction_start|>",
264
- "<|ocr_correction_end|>",
265
- "<|json_scheme_start|>",
266
- "<|json_scheme_end|>",
267
- "<|source_interpretation_start|>",
268
- "<|source_interpretation_end|>",
269
- "<|query_start|>",
270
- "<|query_end|>"
271
  ],
272
  "clean_up_tokenization_spaces": true,
273
  "model_max_length": 1000000000000000019884624838656,
 
32
  "single_word": false,
33
  "special": true
34
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
35
  "65526": {
36
+ "content": "<|query_start|>",
37
  "lstrip": false,
38
  "normalized": false,
39
  "rstrip": false,
 
41
  "special": true
42
  },
43
  "65527": {
44
+ "content": "<|query_end|>",
45
  "lstrip": false,
46
  "normalized": false,
47
  "rstrip": false,
 
49
  "special": true
50
  },
51
  "65528": {
52
+ "content": "<|source_start|>",
53
  "lstrip": false,
54
  "normalized": false,
55
  "rstrip": false,
 
57
  "special": true
58
  },
59
  "65529": {
60
+ "content": "<|source_end|>",
61
  "lstrip": false,
62
  "normalized": false,
63
  "rstrip": false,
 
65
  "special": true
66
  },
67
  "65530": {
68
+ "content": "<|source_id_start|>",
69
  "lstrip": false,
70
  "normalized": false,
71
  "rstrip": false,
 
73
  "special": true
74
  },
75
  "65531": {
76
+ "content": "<|source_id_end|>",
77
  "lstrip": false,
78
  "normalized": false,
79
  "rstrip": false,
 
81
  "special": true
82
  },
83
  "65532": {
84
+ "content": "<|source_analysis_start|>",
85
  "lstrip": false,
86
  "normalized": false,
87
  "rstrip": false,
 
89
  "special": true
90
  },
91
  "65533": {
92
+ "content": "<|source_analysis_end|>",
93
  "lstrip": false,
94
  "normalized": false,
95
  "rstrip": false,
 
97
  "special": true
98
  },
99
  "65534": {
100
+ "content": "<|answer_start|>",
101
  "lstrip": false,
102
  "normalized": false,
103
  "rstrip": false,
 
105
  "special": true
106
  },
107
  "65535": {
108
+ "content": "<|answer_end|>",
109
  "lstrip": false,
110
  "normalized": false,
111
  "rstrip": false,
 
114
  }
115
  },
116
  "additional_special_tokens": [
117
+ "<|query_start|>",
118
+ "<|query_end|>",
 
 
 
 
119
  "<|source_start|>",
120
  "<|source_end|>",
121
+ "<|source_id_start|>",
122
+ "<|source_id_end|>",
123
+ "<|source_analysis_start|>",
124
+ "<|source_analysis_end|>",
125
  "<|answer_start|>",
126
+ "<|answer_end|>"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
127
  ],
128
  "clean_up_tokenization_spaces": true,
129
  "model_max_length": 1000000000000000019884624838656,