Add tiny Fuyu tokenizer fixture for testing
Browse files- tokenizer.json +6 -2
tokenizer.json
CHANGED
|
@@ -13,7 +13,7 @@
|
|
| 13 |
"special": true
|
| 14 |
},
|
| 15 |
{
|
| 16 |
-
"id":
|
| 17 |
"content": "|ENDOFTEXT|",
|
| 18 |
"single_word": false,
|
| 19 |
"lstrip": false,
|
|
@@ -85,7 +85,7 @@
|
|
| 85 |
"|ENDOFTEXT|": {
|
| 86 |
"id": "|ENDOFTEXT|",
|
| 87 |
"ids": [
|
| 88 |
-
|
| 89 |
],
|
| 90 |
"tokens": [
|
| 91 |
"|ENDOFTEXT|"
|
|
@@ -2129,6 +2129,10 @@
|
|
| 2129 |
"|NEWLINE|",
|
| 2130 |
0.0
|
| 2131 |
],
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2132 |
[
|
| 2133 |
"|ENDOFTEXT|",
|
| 2134 |
0.0
|
|
|
|
| 13 |
"special": true
|
| 14 |
},
|
| 15 |
{
|
| 16 |
+
"id": 503,
|
| 17 |
"content": "|ENDOFTEXT|",
|
| 18 |
"single_word": false,
|
| 19 |
"lstrip": false,
|
|
|
|
| 85 |
"|ENDOFTEXT|": {
|
| 86 |
"id": "|ENDOFTEXT|",
|
| 87 |
"ids": [
|
| 88 |
+
503
|
| 89 |
],
|
| 90 |
"tokens": [
|
| 91 |
"|ENDOFTEXT|"
|
|
|
|
| 2129 |
"|NEWLINE|",
|
| 2130 |
0.0
|
| 2131 |
],
|
| 2132 |
+
[
|
| 2133 |
+
"▁",
|
| 2134 |
+
-1.457676649093628
|
| 2135 |
+
],
|
| 2136 |
[
|
| 2137 |
"|ENDOFTEXT|",
|
| 2138 |
0.0
|