Hugging Face's logo

naot97
/
my-tokenizer

Transformers
Model card Files
xet
 Community
naot97 commited on
Commit
f5a03c6
·
verified ·
1 Parent(s): 6efda3d

Upload tokenizer

Browse files
Files changed (5) hide show
  1. added_tokens.json +0 -0
  2. special_tokens_map.json +12 -0
  3. tokenizer.json +2 -2
  4. tokenizer.model +2 -2
  5. tokenizer_config.json +0 -0
added_tokens.json CHANGED
The diff for this file is too large to render. See raw diff
 
special_tokens_map.json CHANGED
@@ -1,4 +1,9 @@
1
  {
 
 
 
 
 
2
  "bos_token": {
3
  "content": "<s>",
4
  "lstrip": false,
@@ -13,6 +18,13 @@
13
  "rstrip": false,
14
  "single_word": false
15
  },
 
 
 
 
 
 
 
16
  "unk_token": {
17
  "content": "<unk>",
18
  "lstrip": false,
 
1
  {
2
+ "additional_special_tokens": [
3
+ "<unk>",
4
+ "<s>",
5
+ "</s>"
6
+ ],
7
  "bos_token": {
8
  "content": "<s>",
9
  "lstrip": false,
 
18
  "rstrip": false,
19
  "single_word": false
20
  },
21
+ "pad_token": {
22
+ "content": "<unk>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false
27
+ },
28
  "unk_token": {
29
  "content": "<unk>",
30
  "lstrip": false,
tokenizer.json CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:881b4bc6c2b29cb38c591cadc1f461969e26cfc819a8a89d9f40ba89e4f84a5f
3
- size 5457204
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d4d937e4496ffe58ac7a518dce7159d79aac8860933b801f3dff298bb671aa21
3
+ size 8197260
tokenizer.model CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
3
- size 493443
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e792a804bbfc19a96b61b87109b8f2b0b7c92830025f285b402ba27c0c309c6f
3
+ size 596883
tokenizer_config.json CHANGED
The diff for this file is too large to render. See raw diff