Hugging Face's logo

ncardus
/
charTokenizerModel

Model card Files
xet
 Community
ncardus commited on
Commit
6925749
·
1 Parent(s): 0da83ff

Upload tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer.json +10 -10
  2. tokenizer_config.json +4 -4
tokenizer.json CHANGED
@@ -5,7 +5,7 @@
5
  "added_tokens": [
6
  {
7
  "id": 0,
8
- "content": "[UNK]",
9
  "single_word": false,
10
  "lstrip": false,
11
  "rstrip": false,
@@ -14,7 +14,7 @@
14
  },
15
  {
16
  "id": 1,
17
- "content": "[PAD]",
18
  "single_word": false,
19
  "lstrip": false,
20
  "rstrip": false,
@@ -23,7 +23,7 @@
23
  },
24
  {
25
  "id": 2,
26
- "content": "[CLS]",
27
  "single_word": false,
28
  "lstrip": false,
29
  "rstrip": false,
@@ -32,7 +32,7 @@
32
  },
33
  {
34
  "id": 3,
35
- "content": "[SEP]",
36
  "single_word": false,
37
  "lstrip": false,
38
  "rstrip": false,
@@ -76,11 +76,11 @@
76
  "type": "BertProcessing",
77
  "sep": [
78
  "[SEP]",
79
- 0
80
  ],
81
  "cls": [
82
  "[CLS]",
83
- 1
84
  ]
85
  },
86
  "decoder": null,
@@ -93,10 +93,10 @@
93
  "fuse_unk": false,
94
  "byte_fallback": false,
95
  "vocab": {
96
- "[UNK]": 0,
97
- "[PAD]": 1,
98
- "[CLS]": 2,
99
- "[SEP]": 3,
100
  "[MASK]": 4,
101
  "'": 5,
102
  "(": 6,
 
5
  "added_tokens": [
6
  {
7
  "id": 0,
8
+ "content": "[CLS]",
9
  "single_word": false,
10
  "lstrip": false,
11
  "rstrip": false,
 
14
  },
15
  {
16
  "id": 1,
17
+ "content": "[SEP]",
18
  "single_word": false,
19
  "lstrip": false,
20
  "rstrip": false,
 
23
  },
24
  {
25
  "id": 2,
26
+ "content": "[UNK]",
27
  "single_word": false,
28
  "lstrip": false,
29
  "rstrip": false,
 
32
  },
33
  {
34
  "id": 3,
35
+ "content": "[PAD]",
36
  "single_word": false,
37
  "lstrip": false,
38
  "rstrip": false,
 
76
  "type": "BertProcessing",
77
  "sep": [
78
  "[SEP]",
79
+ 1
80
  ],
81
  "cls": [
82
  "[CLS]",
83
+ 0
84
  ]
85
  },
86
  "decoder": null,
 
93
  "fuse_unk": false,
94
  "byte_fallback": false,
95
  "vocab": {
96
+ "[CLS]": 0,
97
+ "[SEP]": 1,
98
+ "[UNK]": 2,
99
+ "[PAD]": 3,
100
  "[MASK]": 4,
101
  "'": 5,
102
  "(": 6,
tokenizer_config.json CHANGED
@@ -1,7 +1,7 @@
1
  {
2
  "added_tokens_decoder": {
3
  "0": {
4
- "content": "[UNK]",
5
  "lstrip": false,
6
  "normalized": false,
7
  "rstrip": false,
@@ -9,7 +9,7 @@
9
  "special": true
10
  },
11
  "1": {
12
- "content": "[PAD]",
13
  "lstrip": false,
14
  "normalized": false,
15
  "rstrip": false,
@@ -17,7 +17,7 @@
17
  "special": true
18
  },
19
  "2": {
20
- "content": "[CLS]",
21
  "lstrip": false,
22
  "normalized": false,
23
  "rstrip": false,
@@ -25,7 +25,7 @@
25
  "special": true
26
  },
27
  "3": {
28
- "content": "[SEP]",
29
  "lstrip": false,
30
  "normalized": false,
31
  "rstrip": false,
 
1
  {
2
  "added_tokens_decoder": {
3
  "0": {
4
+ "content": "[CLS]",
5
  "lstrip": false,
6
  "normalized": false,
7
  "rstrip": false,
 
9
  "special": true
10
  },
11
  "1": {
12
+ "content": "[SEP]",
13
  "lstrip": false,
14
  "normalized": false,
15
  "rstrip": false,
 
17
  "special": true
18
  },
19
  "2": {
20
+ "content": "[UNK]",
21
  "lstrip": false,
22
  "normalized": false,
23
  "rstrip": false,
 
25
  "special": true
26
  },
27
  "3": {
28
+ "content": "[PAD]",
29
  "lstrip": false,
30
  "normalized": false,
31
  "rstrip": false,