Commit ·
62357a7
1
Parent(s): bc4ddeb
Upload tokenizer
Browse files- tokenizer.json +0 -9
- tokenizer.model +1 -1
tokenizer.json
CHANGED
|
@@ -91,15 +91,6 @@
|
|
| 91 |
}
|
| 92 |
],
|
| 93 |
"special_tokens": {
|
| 94 |
-
"</s>": {
|
| 95 |
-
"id": "</s>",
|
| 96 |
-
"ids": [
|
| 97 |
-
2
|
| 98 |
-
],
|
| 99 |
-
"tokens": [
|
| 100 |
-
"</s>"
|
| 101 |
-
]
|
| 102 |
-
},
|
| 103 |
"<s>": {
|
| 104 |
"id": "<s>",
|
| 105 |
"ids": [
|
|
|
|
| 91 |
}
|
| 92 |
],
|
| 93 |
"special_tokens": {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 94 |
"<s>": {
|
| 95 |
"id": "<s>",
|
| 96 |
"ids": [
|
tokenizer.model
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 534194
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:81c4a3c9a9bbad64636d93660b6982940cec979a398f42684ba7194d118a3f21
|
| 3 |
size 534194
|