Hugging Face's logo

olaverse
/
naija-embed-base

Sentence Similarity
sentence-transformers
Safetensors
Hausa
Yoruba
Igbo
modernbert
feature-extraction
nigerian
hausa
yoruba
igbo
cross-lingual
text-embeddings-inference
Model card Files
xet
 Community
olumideola commited on
Commit
ba76fcd
·
verified ·
1 Parent(s): 5a22d35

Upload tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer.json +51 -2
  2. tokenizer_config.json +5 -12
tokenizer.json CHANGED
@@ -106,28 +106,77 @@
106
  "post_processor": {
107
  "type": "TemplateProcessing",
108
  "single": [
 
 
 
 
 
 
109
  {
110
  "Sequence": {
111
  "id": "A",
112
  "type_id": 0
113
  }
 
 
 
 
 
 
114
  }
115
  ],
116
  "pair": [
 
 
 
 
 
 
117
  {
118
  "Sequence": {
119
  "id": "A",
120
  "type_id": 0
121
  }
122
  },
 
 
 
 
 
 
123
  {
124
  "Sequence": {
125
  "id": "B",
126
- "type_id": 1
 
 
 
 
 
 
127
  }
128
  }
129
  ],
130
- "special_tokens": {}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
131
  },
132
  "decoder": {
133
  "type": "ByteLevel",
 
106
  "post_processor": {
107
  "type": "TemplateProcessing",
108
  "single": [
109
+ {
110
+ "SpecialToken": {
111
+ "id": "[CLS]",
112
+ "type_id": 0
113
+ }
114
+ },
115
  {
116
  "Sequence": {
117
  "id": "A",
118
  "type_id": 0
119
  }
120
+ },
121
+ {
122
+ "SpecialToken": {
123
+ "id": "[SEP]",
124
+ "type_id": 0
125
+ }
126
  }
127
  ],
128
  "pair": [
129
+ {
130
+ "SpecialToken": {
131
+ "id": "[CLS]",
132
+ "type_id": 0
133
+ }
134
+ },
135
  {
136
  "Sequence": {
137
  "id": "A",
138
  "type_id": 0
139
  }
140
  },
141
+ {
142
+ "SpecialToken": {
143
+ "id": "[SEP]",
144
+ "type_id": 0
145
+ }
146
+ },
147
  {
148
  "Sequence": {
149
  "id": "B",
150
+ "type_id": 0
151
+ }
152
+ },
153
+ {
154
+ "SpecialToken": {
155
+ "id": "[SEP]",
156
+ "type_id": 0
157
  }
158
  }
159
  ],
160
+ "special_tokens": {
161
+ "[CLS]": {
162
+ "id": "[CLS]",
163
+ "ids": [
164
+ 1
165
+ ],
166
+ "tokens": [
167
+ "[CLS]"
168
+ ]
169
+ },
170
+ "[SEP]": {
171
+ "id": "[SEP]",
172
+ "ids": [
173
+ 2
174
+ ],
175
+ "tokens": [
176
+ "[SEP]"
177
+ ]
178
+ }
179
+ }
180
  },
181
  "decoder": {
182
  "type": "ByteLevel",
tokenizer_config.json CHANGED
@@ -1,17 +1,10 @@
1
  {
2
  "backend": "tokenizers",
3
- "bos_token": "<s>",
4
- "eos_token": "</s>",
5
- "mask_token": "<mask>",
6
- "max_length": 128,
7
  "model_max_length": 1000000000000000019884624838656,
8
- "pad_to_multiple_of": null,
9
- "pad_token": "<pad>",
10
- "pad_token_type_id": 0,
11
- "padding_side": "right",
12
- "stride": 0,
13
  "tokenizer_class": "TokenizersBackend",
14
- "truncation_side": "right",
15
- "truncation_strategy": "longest_first",
16
- "unk_token": "<unk>"
17
  }
 
1
  {
2
  "backend": "tokenizers",
3
+ "cls_token": "[CLS]",
4
+ "mask_token": "[MASK]",
 
 
5
  "model_max_length": 1000000000000000019884624838656,
6
+ "pad_token": "[PAD]",
7
+ "sep_token": "[SEP]",
 
 
 
8
  "tokenizer_class": "TokenizersBackend",
9
+ "unk_token": "[UNK]"
 
 
10
  }