Add tiny Fuyu tokenizer fixture for testing
Browse files- tokenizer.json +10 -2
tokenizer.json
CHANGED
|
@@ -13,7 +13,7 @@
|
|
| 13 |
"special": true
|
| 14 |
},
|
| 15 |
{
|
| 16 |
-
"id":
|
| 17 |
"content": "|ENDOFTEXT|",
|
| 18 |
"single_word": false,
|
| 19 |
"lstrip": false,
|
|
@@ -85,7 +85,7 @@
|
|
| 85 |
"|ENDOFTEXT|": {
|
| 86 |
"id": "|ENDOFTEXT|",
|
| 87 |
"ids": [
|
| 88 |
-
|
| 89 |
],
|
| 90 |
"tokens": [
|
| 91 |
"|ENDOFTEXT|"
|
|
@@ -2121,6 +2121,14 @@
|
|
| 2121 |
"jacjEvKbBD",
|
| 2122 |
0.0
|
| 2123 |
],
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2124 |
[
|
| 2125 |
"|ENDOFTEXT|",
|
| 2126 |
0.0
|
|
|
|
| 13 |
"special": true
|
| 14 |
},
|
| 15 |
{
|
| 16 |
+
"id": 502,
|
| 17 |
"content": "|ENDOFTEXT|",
|
| 18 |
"single_word": false,
|
| 19 |
"lstrip": false,
|
|
|
|
| 85 |
"|ENDOFTEXT|": {
|
| 86 |
"id": "|ENDOFTEXT|",
|
| 87 |
"ids": [
|
| 88 |
+
502
|
| 89 |
],
|
| 90 |
"tokens": [
|
| 91 |
"|ENDOFTEXT|"
|
|
|
|
| 2121 |
"jacjEvKbBD",
|
| 2122 |
0.0
|
| 2123 |
],
|
| 2124 |
+
[
|
| 2125 |
+
"|SPEAKER|",
|
| 2126 |
+
0.0
|
| 2127 |
+
],
|
| 2128 |
+
[
|
| 2129 |
+
"|NEWLINE|",
|
| 2130 |
+
0.0
|
| 2131 |
+
],
|
| 2132 |
[
|
| 2133 |
"|ENDOFTEXT|",
|
| 2134 |
0.0
|