Hugging Face's logo

Marxav
/
frpron

Text Generation
Transformers
PyTorch
French
gpt2
text-generation-inference
Model card Files
xet
Community
1
Marxav commited on
Commit
d41cb5e
·
1 Parent(s): dade576

add tokenizer

Browse files
Files changed (3) hide show
  1. added_tokens.json +1 -1
  2. tokenizer.json +264 -252
  3. vocab.json +1 -1
added_tokens.json CHANGED
@@ -1 +1 @@
1
- {"<": 254, ">": 253, "#": 255}
 
1
+ {"<": 266, ">": 265, "#": 267}
tokenizer.json CHANGED
@@ -13,7 +13,7 @@
13
  "special": true
14
  },
15
  {
16
- "id": 253,
17
  "content": ">",
18
  "single_word": false,
19
  "lstrip": false,
@@ -22,7 +22,7 @@
22
  "special": true
23
  },
24
  {
25
- "id": 254,
26
  "content": "<",
27
  "single_word": false,
28
  "lstrip": false,
@@ -31,7 +31,7 @@
31
  "special": true
32
  },
33
  {
34
- "id": 255,
35
  "content": "#",
36
  "single_word": false,
37
  "lstrip": false,
@@ -68,255 +68,267 @@
68
  "\n": 1,
69
  " ": 2,
70
  "!": 3,
71
- "&": 4,
72
- "'": 5,
73
- "(": 6,
74
- ")": 7,
75
- "*": 8,
76
- "+": 9,
77
- "-": 10,
78
- ".": 11,
79
- "/": 12,
80
- "0": 13,
81
- "1": 14,
82
- "2": 15,
83
- "3": 16,
84
- "4": 17,
85
- "5": 18,
86
- "6": 19,
87
- "7": 20,
88
- "8": 21,
89
- "9": 22,
90
- ":": 23,
91
- ";": 24,
92
- "@": 25,
93
- "A": 26,
94
- "B": 27,
95
- "C": 28,
96
- "D": 29,
97
- "E": 30,
98
- "F": 31,
99
- "G": 32,
100
- "H": 33,
101
- "I": 34,
102
- "J": 35,
103
- "K": 36,
104
- "L": 37,
105
- "M": 38,
106
- "N": 39,
107
- "O": 40,
108
- "P": 41,
109
- "Q": 42,
110
- "R": 43,
111
- "S": 44,
112
- "T": 45,
113
- "U": 46,
114
- "V": 47,
115
- "W": 48,
116
- "X": 49,
117
- "Y": 50,
118
- "Z": 51,
119
- "a": 52,
120
- "b": 53,
121
- "c": 54,
122
- "d": 55,
123
- "e": 56,
124
- "f": 57,
125
- "g": 58,
126
- "h": 59,
127
- "i": 60,
128
- "j": 61,
129
- "k": 62,
130
- "l": 63,
131
- "m": 64,
132
- "n": 65,
133
- "o": 66,
134
- "p": 67,
135
- "q": 68,
136
- "r": 69,
137
- "s": 70,
138
- "t": 71,
139
- "u": 72,
140
- "v": 73,
141
- "w": 74,
142
- "x": 75,
143
- "y": 76,
144
- "z": 77,
145
- "¡": 78,
146
- "¢": 79,
147
- "£": 80,
148
- "¤": 81,
149
- "¥": 82,
150
- "¦": 83,
151
- "§": 84,
152
- "¨": 85,
153
- "©": 86,
154
- "ª": 87,
155
- "«": 88,
156
- "¬": 89,
157
- "®": 90,
158
- "¯": 91,
159
- "°": 92,
160
- "±": 93,
161
- "²": 94,
162
- "³": 95,
163
- "´": 96,
164
- "µ": 97,
165
- "": 98,
166
- "·": 99,
167
- "¸": 100,
168
- "¹": 101,
169
- "º": 102,
170
- "»": 103,
171
- "¼": 104,
172
- "½": 105,
173
- "¾": 106,
174
- "¿": 107,
175
- "Á": 108,
176
- "Â": 109,
177
- "Ã": 110,
178
- "Ä": 111,
179
- "Å": 112,
180
- "Æ": 113,
181
- "Ç": 114,
182
- "È": 115,
183
- "É": 116,
184
- "Ê": 117,
185
- "Ì": 118,
186
- "Î": 119,
187
- "Ï": 120,
188
- "Ð": 121,
189
- "Ô": 122,
190
- "Ö": 123,
191
- "×": 124,
192
- "Ü": 125,
193
- "ß": 126,
194
- "à": 127,
195
- "á": 128,
196
- "â": 129,
197
- "ã": 130,
198
- "ä": 131,
199
- "å": 132,
200
- "æ": 133,
201
- "ç": 134,
202
- "è": 135,
203
- "é": 136,
204
- "ê": 137,
205
- "ë": 138,
206
- "ì": 139,
207
- "í": 140,
208
- "î": 141,
209
- "ï": 142,
210
- "ñ": 143,
211
- "ò": 144,
212
- "ó": 145,
213
- "ô": 146,
214
- "õ": 147,
215
- "ö": 148,
216
- "ø": 149,
217
- "ù": 150,
218
- "ú": 151,
219
- "û": 152,
220
- "ü": 153,
221
- "ý": 154,
222
- "ÿ": 155,
223
- "ā": 156,
224
- "ć": 157,
225
- "č": 158,
226
- "Ğ": 159,
227
- "ğ": 160,
228
- "Ġ": 161,
229
- "Ģ": 162,
230
- "ģ": 163,
231
- "Ĥ": 164,
232
- "ĥ": 165,
233
- "Ĩ": 166,
234
- "ĩ": 167,
235
- "Ī": 168,
236
- "ī": 169,
237
- "Ĭ": 170,
238
- "ĭ": 171,
239
- "į": 172,
240
- "İ": 173,
241
- "ı": 174,
242
- "IJ": 175,
243
- "ij": 176,
244
- "Ĵ": 177,
245
- "ĵ": 178,
246
- "Ķ": 179,
247
- "ĸ": 180,
248
- "Ĺ": 181,
249
- "ĺ": 182,
250
- "Ļ": 183,
251
- "ļ": 184,
252
- "Ľ": 185,
253
- "ľ": 186,
254
- "Ŀ": 187,
255
- "ŀ": 188,
256
- "Ł": 189,
257
- "ł": 190,
258
- "Ń": 191,
259
- "ŋ": 192,
260
- "ō": 193,
261
- "Œ": 194,
262
- "œ": 195,
263
- "Ś": 196,
264
- "ŝ": 197,
265
- "Ş": 198,
266
- "ş": 199,
267
- "š": 200,
268
- "ū": 201,
269
- "ŵ": 202,
270
- "ź": 203,
271
- "Ž": 204,
272
- "ž": 205,
273
- "ǘ": 206,
274
- "ȝ": 207,
275
- "ɑ": 208,
276
- "ɔ": 209,
277
- "ə": 210,
278
- "ɛ": 211,
279
- "ɡ": 212,
280
- "ɣ": 213,
281
- "ɥ": 214,
282
- "ɨ": 215,
283
- "ɩ": 216,
284
- "ɲ": 217,
285
- "ʁ": 218,
286
- "ʃ": 219,
287
- "ʋ": 220,
288
- "ʒ": 221,
289
- "ʔ": 222,
290
- "ʻ": 223,
291
- "ʼ": 224,
292
- "ʾ": 225,
293
- "ʿ": 226,
294
- "́": 227,
295
- "̃": 228,
296
- "̐": 229,
297
- "̓": 230,
298
- "͠": 231,
299
- "Δ": 232,
300
- "α": 233,
301
- "β": 234,
302
- "μ": 235,
303
- "σ": 236,
304
- "а": 237,
305
- "е": 238,
306
- "ḷ": 239,
307
- "ṃ": 240,
308
- "ṇ": 241,
309
- "ṣ": 242,
310
- "ṭ": 243,
311
- "–": 244,
312
- "‘": 245,
313
- "’": 246,
314
- "′": 247,
315
- "‿": 248,
316
- "₂": 249,
317
- "€": 250,
318
- "−": 251,
319
- "∴": 252
 
 
 
 
 
 
 
 
 
 
 
 
320
  },
321
  "merges": []
322
  }
 
13
  "special": true
14
  },
15
  {
16
+ "id": 265,
17
  "content": ">",
18
  "single_word": false,
19
  "lstrip": false,
 
22
  "special": true
23
  },
24
  {
25
+ "id": 266,
26
  "content": "<",
27
  "single_word": false,
28
  "lstrip": false,
 
31
  "special": true
32
  },
33
  {
34
+ "id": 267,
35
  "content": "#",
36
  "single_word": false,
37
  "lstrip": false,
 
68
  "\n": 1,
69
  " ": 2,
70
  "!": 3,
71
+ "%": 4,
72
+ "&": 5,
73
+ "'": 6,
74
+ "(": 7,
75
+ ")": 8,
76
+ "*": 9,
77
+ "+": 10,
78
+ ",": 11,
79
+ "-": 12,
80
+ ".": 13,
81
+ "/": 14,
82
+ "0": 15,
83
+ "1": 16,
84
+ "2": 17,
85
+ "3": 18,
86
+ "4": 19,
87
+ "5": 20,
88
+ "6": 21,
89
+ "7": 22,
90
+ "8": 23,
91
+ "9": 24,
92
+ ":": 25,
93
+ ";": 26,
94
+ "=": 27,
95
+ "?": 28,
96
+ "@": 29,
97
+ "A": 30,
98
+ "B": 31,
99
+ "C": 32,
100
+ "D": 33,
101
+ "E": 34,
102
+ "F": 35,
103
+ "G": 36,
104
+ "H": 37,
105
+ "I": 38,
106
+ "J": 39,
107
+ "K": 40,
108
+ "L": 41,
109
+ "M": 42,
110
+ "N": 43,
111
+ "O": 44,
112
+ "P": 45,
113
+ "Q": 46,
114
+ "R": 47,
115
+ "S": 48,
116
+ "T": 49,
117
+ "U": 50,
118
+ "V": 51,
119
+ "W": 52,
120
+ "X": 53,
121
+ "Y": 54,
122
+ "Z": 55,
123
+ "\\": 56,
124
+ "]": 57,
125
+ "a": 58,
126
+ "b": 59,
127
+ "c": 60,
128
+ "d": 61,
129
+ "e": 62,
130
+ "f": 63,
131
+ "g": 64,
132
+ "h": 65,
133
+ "i": 66,
134
+ "j": 67,
135
+ "k": 68,
136
+ "l": 69,
137
+ "m": 70,
138
+ "n": 71,
139
+ "o": 72,
140
+ "p": 73,
141
+ "q": 74,
142
+ "r": 75,
143
+ "s": 76,
144
+ "t": 77,
145
+ "u": 78,
146
+ "v": 79,
147
+ "w": 80,
148
+ "x": 81,
149
+ "y": 82,
150
+ "z": 83,
151
+ "}": 84,
152
+ "¡": 85,
153
+ "¢": 86,
154
+ "£": 87,
155
+ "¤": 88,
156
+ "¥": 89,
157
+ "¦": 90,
158
+ "§": 91,
159
+ "¨": 92,
160
+ "©": 93,
161
+ "ª": 94,
162
+ "«": 95,
163
+ "¬": 96,
164
+ "®": 97,
165
+ "¯": 98,
166
+ "°": 99,
167
+ "±": 100,
168
+ "²": 101,
169
+ "³": 102,
170
+ "´": 103,
171
+ "µ": 104,
172
+ "": 105,
173
+ "·": 106,
174
+ "¸": 107,
175
+ "¹": 108,
176
+ "º": 109,
177
+ "»": 110,
178
+ "¼": 111,
179
+ "½": 112,
180
+ "¾": 113,
181
+ "¿": 114,
182
+ "Á": 115,
183
+ "Â": 116,
184
+ "Ã": 117,
185
+ "Ä": 118,
186
+ "Å": 119,
187
+ "Æ": 120,
188
+ "Ç": 121,
189
+ "È": 122,
190
+ "É": 123,
191
+ "Ê": 124,
192
+ "Ë": 125,
193
+ "Ì": 126,
194
+ "Í": 127,
195
+ "Î": 128,
196
+ "Ï": 129,
197
+ "Ð": 130,
198
+ "Ô": 131,
199
+ "Ö": 132,
200
+ "×": 133,
201
+ "Ü": 134,
202
+ "ß": 135,
203
+ "à": 136,
204
+ "á": 137,
205
+ "â": 138,
206
+ "ã": 139,
207
+ "ä": 140,
208
+ "å": 141,
209
+ "æ": 142,
210
+ "ç": 143,
211
+ "è": 144,
212
+ "é": 145,
213
+ "ê": 146,
214
+ "ë": 147,
215
+ "ì": 148,
216
+ "í": 149,
217
+ "î": 150,
218
+ "ï": 151,
219
+ "ñ": 152,
220
+ "ò": 153,
221
+ "ó": 154,
222
+ "ô": 155,
223
+ "õ": 156,
224
+ "ö": 157,
225
+ "ø": 158,
226
+ "ù": 159,
227
+ "ú": 160,
228
+ "û": 161,
229
+ "ü": 162,
230
+ "ý": 163,
231
+ "ÿ": 164,
232
+ "ā": 165,
233
+ "ć": 166,
234
+ "č": 167,
235
+ "Ğ": 168,
236
+ "ğ": 169,
237
+ "Ġ": 170,
238
+ "Ģ": 171,
239
+ "ģ": 172,
240
+ "Ĥ": 173,
241
+ "ĥ": 174,
242
+ "Ħ": 175,
243
+ "ħ": 176,
244
+ "Ĩ": 177,
245
+ "ĩ": 178,
246
+ "Ī": 179,
247
+ "ī": 180,
248
+ "Ĭ": 181,
249
+ "ĭ": 182,
250
+ "Į": 183,
251
+ "į": 184,
252
+ "İ": 185,
253
+ "ı": 186,
254
+ "IJ": 187,
255
+ "ij": 188,
256
+ "Ĵ": 189,
257
+ "ĵ": 190,
258
+ "Ķ": 191,
259
+ "ĸ": 192,
260
+ "Ĺ": 193,
261
+ "ĺ": 194,
262
+ "Ļ": 195,
263
+ "ļ": 196,
264
+ "Ľ": 197,
265
+ "ľ": 198,
266
+ "Ŀ": 199,
267
+ "ŀ": 200,
268
+ "Ł": 201,
269
+ "ł": 202,
270
+ "Ń": 203,
271
+ "ŋ": 204,
272
+ "ō": 205,
273
+ "Œ": 206,
274
+ "œ": 207,
275
+ "Ś": 208,
276
+ "ŝ": 209,
277
+ "Ş": 210,
278
+ "ş": 211,
279
+ "š": 212,
280
+ "ū": 213,
281
+ "ŵ": 214,
282
+ "ź": 215,
283
+ "Ž": 216,
284
+ "ž": 217,
285
+ "ǘ": 218,
286
+ "ȝ": 219,
287
+ "ɑ": 220,
288
+ "ɔ": 221,
289
+ "ə": 222,
290
+ "ɛ": 223,
291
+ "ɡ": 224,
292
+ "ɣ": 225,
293
+ "ɥ": 226,
294
+ "ɨ": 227,
295
+ "ɩ": 228,
296
+ "ɲ": 229,
297
+ "ʁ": 230,
298
+ "ʃ": 231,
299
+ "ʋ": 232,
300
+ "ʒ": 233,
301
+ "ʔ": 234,
302
+ "ʻ": 235,
303
+ "ʼ": 236,
304
+ "ʾ": 237,
305
+ "ʿ": 238,
306
+ "́": 239,
307
+ "̃": 240,
308
+ "̐": 241,
309
+ "̓": 242,
310
+ "͠": 243,
311
+ "Δ": 244,
312
+ "α": 245,
313
+ "β": 246,
314
+ "μ": 247,
315
+ "σ": 248,
316
+ "а": 249,
317
+ "е": 250,
318
+ "ḷ": 251,
319
+ "ṃ": 252,
320
+ "ṇ": 253,
321
+ "ṣ": 254,
322
+ "ṭ": 255,
323
+ "–": 256,
324
+ "‘": 257,
325
+ "’": 258,
326
+ "′": 259,
327
+ "‿": 260,
328
+ "₂": 261,
329
+ "€": 262,
330
+ "−": 263,
331
+ "∴": 264
332
  },
333
  "merges": []
334
  }
vocab.json CHANGED
@@ -1 +1 @@
1
- {"<|endoftext|>":0,"\n":1," ":2,"!":3,"&":4,"'":5,"(":6,")":7,"*":8,"+":9,"-":10,".":11,"/":12,"0":13,"1":14,"2":15,"3":16,"4":17,"5":18,"6":19,"7":20,"8":21,"9":22,":":23,";":24,"@":25,"A":26,"B":27,"C":28,"D":29,"E":30,"F":31,"G":32,"H":33,"I":34,"J":35,"K":36,"L":37,"M":38,"N":39,"O":40,"P":41,"Q":42,"R":43,"S":44,"T":45,"U":46,"V":47,"W":48,"X":49,"Y":50,"Z":51,"a":52,"b":53,"c":54,"d":55,"e":56,"f":57,"g":58,"h":59,"i":60,"j":61,"k":62,"l":63,"m":64,"n":65,"o":66,"p":67,"q":68,"r":69,"s":70,"t":71,"u":72,"v":73,"w":74,"x":75,"y":76,"z":77,"¡":78,"¢":79,"£":80,"¤":81,"¥":82,"¦":83,"§":84,"¨":85,"©":86,"ª":87,"«":88,"¬":89,"®":90,"¯":91,"°":92,"±":93,"²":94,"³":95,"´":96,"µ":97,"":98,"·":99,"¸":100,"¹":101,"º":102,"»":103,"¼":104,"½":105,"¾":106,"¿":107,"Á":108,"Â":109,"Ã":110,"Ä":111,"Å":112,"Æ":113,"Ç":114,"È":115,"É":116,"Ê":117,"Ì":118,"Î":119,"Ï":120,"Ð":121,"Ô":122,"Ö":123,"×":124,"Ü":125,"ß":126,"à":127,"á":128,"â":129,"ã":130,"ä":131,"å":132,"æ":133,"ç":134,"è":135,"é":136,"ê":137,"ë":138,"ì":139,"í":140,"î":141,"ï":142,"ñ":143,"ò":144,"ó":145,"ô":146,"õ":147,"ö":148,"ø":149,"ù":150,"ú":151,"û":152,"ü":153,"ý":154,"ÿ":155,"ā":156,"ć":157,"č":158,"Ğ":159,"ğ":160,"Ġ":161,"Ģ":162,"ģ":163,"Ĥ":164,"ĥ":165,"Ĩ":166,"ĩ":167,"Ī":168,"ī":169,"Ĭ":170,"ĭ":171,"į":172,"İ":173,"ı":174,"IJ":175,"ij":176,"Ĵ":177,"ĵ":178,"Ķ":179,"ĸ":180,"Ĺ":181,"ĺ":182,"Ļ":183,"ļ":184,"Ľ":185,"ľ":186,"Ŀ":187,"ŀ":188,"Ł":189,"ł":190,"Ń":191,"ŋ":192,"ō":193,"Œ":194,"œ":195,"Ś":196,"ŝ":197,"Ş":198,"ş":199,"š":200,"ū":201,"ŵ":202,"ź":203,"Ž":204,"ž":205,"ǘ":206,"ȝ":207,"ɑ":208,"ɔ":209,"ə":210,"ɛ":211,"ɡ":212,"ɣ":213,"ɥ":214,"ɨ":215,"ɩ":216,"ɲ":217,"ʁ":218,"ʃ":219,"ʋ":220,"ʒ":221,"ʔ":222,"ʻ":223,"ʼ":224,"ʾ":225,"ʿ":226,"́":227,"̃":228,"̐":229,"̓":230,"͠":231,"Δ":232,"α":233,"β":234,"μ":235,"σ":236,"а":237,"е":238,"ḷ":239,"ṃ":240,"ṇ":241,"ṣ":242,"ṭ":243,"–":244,"‘":245,"’":246,"′":247,"‿":248,"₂":249,"€":250,"−":251,"∴":252}
 
1
+ {"<|endoftext|>":0,"\n":1," ":2,"!":3,"%":4,"&":5,"'":6,"(":7,")":8,"*":9,"+":10,",":11,"-":12,".":13,"/":14,"0":15,"1":16,"2":17,"3":18,"4":19,"5":20,"6":21,"7":22,"8":23,"9":24,":":25,";":26,"=":27,"?":28,"@":29,"A":30,"B":31,"C":32,"D":33,"E":34,"F":35,"G":36,"H":37,"I":38,"J":39,"K":40,"L":41,"M":42,"N":43,"O":44,"P":45,"Q":46,"R":47,"S":48,"T":49,"U":50,"V":51,"W":52,"X":53,"Y":54,"Z":55,"\\":56,"]":57,"a":58,"b":59,"c":60,"d":61,"e":62,"f":63,"g":64,"h":65,"i":66,"j":67,"k":68,"l":69,"m":70,"n":71,"o":72,"p":73,"q":74,"r":75,"s":76,"t":77,"u":78,"v":79,"w":80,"x":81,"y":82,"z":83,"}":84,"¡":85,"¢":86,"£":87,"¤":88,"¥":89,"¦":90,"§":91,"¨":92,"©":93,"ª":94,"«":95,"¬":96,"®":97,"¯":98,"°":99,"±":100,"²":101,"³":102,"´":103,"µ":104,"":105,"·":106,"¸":107,"¹":108,"º":109,"»":110,"¼":111,"½":112,"¾":113,"¿":114,"Á":115,"Â":116,"Ã":117,"Ä":118,"Å":119,"Æ":120,"Ç":121,"È":122,"É":123,"Ê":124,"Ë":125,"Ì":126,"Í":127,"Î":128,"Ï":129,"Ð":130,"Ô":131,"Ö":132,"×":133,"Ü":134,"ß":135,"à":136,"á":137,"â":138,"ã":139,"ä":140,"å":141,"æ":142,"ç":143,"è":144,"é":145,"ê":146,"ë":147,"ì":148,"í":149,"î":150,"ï":151,"ñ":152,"ò":153,"ó":154,"ô":155,"õ":156,"ö":157,"ø":158,"ù":159,"ú":160,"û":161,"ü":162,"ý":163,"ÿ":164,"ā":165,"ć":166,"č":167,"Ğ":168,"ğ":169,"Ġ":170,"Ģ":171,"ģ":172,"Ĥ":173,"ĥ":174,"Ħ":175,"ħ":176,"Ĩ":177,"ĩ":178,"Ī":179,"ī":180,"Ĭ":181,"ĭ":182,"Į":183,"į":184,"İ":185,"ı":186,"IJ":187,"ij":188,"Ĵ":189,"ĵ":190,"Ķ":191,"ĸ":192,"Ĺ":193,"ĺ":194,"Ļ":195,"ļ":196,"Ľ":197,"ľ":198,"Ŀ":199,"ŀ":200,"Ł":201,"ł":202,"Ń":203,"ŋ":204,"ō":205,"Œ":206,"œ":207,"Ś":208,"ŝ":209,"Ş":210,"ş":211,"š":212,"ū":213,"ŵ":214,"ź":215,"Ž":216,"ž":217,"ǘ":218,"ȝ":219,"ɑ":220,"ɔ":221,"ə":222,"ɛ":223,"ɡ":224,"ɣ":225,"ɥ":226,"ɨ":227,"ɩ":228,"ɲ":229,"ʁ":230,"ʃ":231,"ʋ":232,"ʒ":233,"ʔ":234,"ʻ":235,"ʼ":236,"ʾ":237,"ʿ":238,"́":239,"̃":240,"̐":241,"̓":242,"͠":243,"Δ":244,"α":245,"β":246,"μ":247,"σ":248,"а":249,"е":250,"ḷ":251,"ṃ":252,"ṇ":253,"ṣ":254,"ṭ":255,"–":256,"‘":257,"’":258,"′":259,"‿":260,"₂":261,"€":262,"−":263,"∴":264}