Delower commited on
Commit
941b22f
·
verified ·
1 Parent(s): 93e1dc6

Upload tokenizer

Browse files
Files changed (3) hide show
  1. special_tokens_map.json +23 -0
  2. tokenizer.json +227 -0
  3. tokenizer_config.json +35 -0
special_tokens_map.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "[BOS]",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "[EOS]",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": {
17
+ "content": "[PAD]",
18
+ "lstrip": false,
19
+ "normalized": false,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ }
23
+ }
tokenizer.json ADDED
@@ -0,0 +1,227 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "version": "1.0",
3
+ "truncation": null,
4
+ "padding": null,
5
+ "added_tokens": [
6
+ {
7
+ "id": 0,
8
+ "content": "[PAD]",
9
+ "single_word": false,
10
+ "lstrip": false,
11
+ "rstrip": false,
12
+ "normalized": false,
13
+ "special": true
14
+ },
15
+ {
16
+ "id": 1,
17
+ "content": "[BOS]",
18
+ "single_word": false,
19
+ "lstrip": false,
20
+ "rstrip": false,
21
+ "normalized": false,
22
+ "special": true
23
+ },
24
+ {
25
+ "id": 2,
26
+ "content": "[EOS]",
27
+ "single_word": false,
28
+ "lstrip": false,
29
+ "rstrip": false,
30
+ "normalized": false,
31
+ "special": true
32
+ }
33
+ ],
34
+ "normalizer": null,
35
+ "pre_tokenizer": {
36
+ "type": "ByteLevel",
37
+ "add_prefix_space": false,
38
+ "trim_offsets": true,
39
+ "use_regex": true
40
+ },
41
+ "post_processor": null,
42
+ "decoder": null,
43
+ "model": {
44
+ "type": "BPE",
45
+ "dropout": null,
46
+ "unk_token": "[PAD]",
47
+ "continuing_subword_prefix": null,
48
+ "end_of_word_suffix": null,
49
+ "fuse_unk": false,
50
+ "byte_fallback": false,
51
+ "ignore_merges": false,
52
+ "vocab": {
53
+ "[PAD]": 0,
54
+ "[BOS]": 1,
55
+ "[EOS]": 2,
56
+ "A": 3,
57
+ "B": 4,
58
+ "C": 5,
59
+ "D": 6,
60
+ "E": 7,
61
+ "F": 8,
62
+ "G": 9,
63
+ "H": 10,
64
+ "I": 11,
65
+ "K": 12,
66
+ "L": 13,
67
+ "M": 14,
68
+ "N": 15,
69
+ "O": 16,
70
+ "P": 17,
71
+ "Q": 18,
72
+ "R": 19,
73
+ "S": 20,
74
+ "T": 21,
75
+ "U": 22,
76
+ "V": 23,
77
+ "W": 24,
78
+ "X": 25,
79
+ "Y": 26,
80
+ "[": 27,
81
+ "]": 28,
82
+ "Ġ": 29,
83
+ "OS": 30,
84
+ "][": 31,
85
+ "AC": 32,
86
+ "AD": 33,
87
+ "BOS": 34,
88
+ "DE": 35,
89
+ "EOS": 36,
90
+ "FG": 37,
91
+ "HI": 38,
92
+ "KL": 39,
93
+ "MN": 40,
94
+ "PQ": 41,
95
+ "PAD": 42,
96
+ "RS": 43,
97
+ "TU": 44,
98
+ "VW": 45,
99
+ "XY": 46,
100
+ "Ġ[": 47,
101
+ "ACDE": 48,
102
+ "FGHI": 49,
103
+ "KLMN": 50,
104
+ "PQRS": 51,
105
+ "TUVW": 52,
106
+ "ACDEFGHI": 53,
107
+ "KLMNPQRS": 54,
108
+ "TUVWXY": 55,
109
+ "ACDEFGHIKLMNPQRS": 56,
110
+ "ACDEFGHIKLMNPQRSTUVWXY": 57
111
+ },
112
+ "merges": [
113
+ [
114
+ "O",
115
+ "S"
116
+ ],
117
+ [
118
+ "]",
119
+ "["
120
+ ],
121
+ [
122
+ "A",
123
+ "C"
124
+ ],
125
+ [
126
+ "A",
127
+ "D"
128
+ ],
129
+ [
130
+ "B",
131
+ "OS"
132
+ ],
133
+ [
134
+ "D",
135
+ "E"
136
+ ],
137
+ [
138
+ "E",
139
+ "OS"
140
+ ],
141
+ [
142
+ "F",
143
+ "G"
144
+ ],
145
+ [
146
+ "H",
147
+ "I"
148
+ ],
149
+ [
150
+ "K",
151
+ "L"
152
+ ],
153
+ [
154
+ "M",
155
+ "N"
156
+ ],
157
+ [
158
+ "P",
159
+ "Q"
160
+ ],
161
+ [
162
+ "P",
163
+ "AD"
164
+ ],
165
+ [
166
+ "R",
167
+ "S"
168
+ ],
169
+ [
170
+ "T",
171
+ "U"
172
+ ],
173
+ [
174
+ "V",
175
+ "W"
176
+ ],
177
+ [
178
+ "X",
179
+ "Y"
180
+ ],
181
+ [
182
+ "Ġ",
183
+ "["
184
+ ],
185
+ [
186
+ "AC",
187
+ "DE"
188
+ ],
189
+ [
190
+ "FG",
191
+ "HI"
192
+ ],
193
+ [
194
+ "KL",
195
+ "MN"
196
+ ],
197
+ [
198
+ "PQ",
199
+ "RS"
200
+ ],
201
+ [
202
+ "TU",
203
+ "VW"
204
+ ],
205
+ [
206
+ "ACDE",
207
+ "FGHI"
208
+ ],
209
+ [
210
+ "KLMN",
211
+ "PQRS"
212
+ ],
213
+ [
214
+ "TUVW",
215
+ "XY"
216
+ ],
217
+ [
218
+ "ACDEFGHI",
219
+ "KLMNPQRS"
220
+ ],
221
+ [
222
+ "ACDEFGHIKLMNPQRS",
223
+ "TUVWXY"
224
+ ]
225
+ ]
226
+ }
227
+ }
tokenizer_config.json ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "[PAD]",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "[BOS]",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "[EOS]",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ }
27
+ },
28
+ "bos_token": "[BOS]",
29
+ "clean_up_tokenization_spaces": false,
30
+ "eos_token": "[EOS]",
31
+ "extra_special_tokens": {},
32
+ "model_max_length": 1000000000000000019884624838656,
33
+ "pad_token": "[PAD]",
34
+ "tokenizer_class": "PreTrainedTokenizer"
35
+ }