Asaf Yehoodai commited on
Commit ·
2f9d3b3
1
Parent(s): 3358534
add tokenizer
Browse files- added_tokens.json +1 -0
- source.spm +0 -0
- special_tokens_map.json +1 -0
- target.spm +0 -0
- tokenizer_config.json +1 -0
- vocab.json +0 -0
added_tokens.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"<ENT0>": 62802, "</ENT0>": 62803, "<ENT1>": 62804, "</ENT1>": 62805, "<ENT2>": 62806, "</ENT2>": 62807, "<ENT3>": 62808, "</ENT3>": 62809, "<ENT4>": 62810, "</ENT4>": 62811, "<ENT5>": 62812, "</ENT5>": 62813, "<ENT6>": 62814, "</ENT6>": 62815, "<ENT7>": 62816, "</ENT7>": 62817, "<ENT8>": 62818, "</ENT8>": 62819, "<ENT9>": 62820, "</ENT9>": 62821, "<ENT10>": 62822, "</ENT10>": 62823, "<ENT11>": 62824, "</ENT11>": 62825, "<ENT12>": 62826, "</ENT12>": 62827, "<ENT13>": 62828, "</ENT13>": 62829, "<ENT14>": 62830, "</ENT14>": 62831, "<ENT15>": 62832, "</ENT15>": 62833, "<ENT16>": 62834, "</ENT16>": 62835, "<ENT17>": 62836, "</ENT17>": 62837, "<ENT18>": 62838, "</ENT18>": 62839, "<ENT19>": 62840, "</ENT19>": 62841, "<MALE>": 62842, "</MALE>": 62843, "<FEMALE>": 62844, "</FEMALE>": 62845}
|
source.spm
ADDED
|
Binary file (801 kB). View file
|
|
|
special_tokens_map.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>"}
|
target.spm
ADDED
|
Binary file (917 kB). View file
|
|
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"source_lang": "eng", "target_lang": "ara", "unk_token": "<unk>", "eos_token": "</s>", "pad_token": "<pad>", "model_max_length": 512, "sp_model_kwargs": {}, "special_tokens_map_file": null, "tokenizer_file": null, "name_or_path": "/cs/labs/oabend/asaf305612939/co-ref-project/mt-output/paper_models/ar/en_ar_gender", "tokenizer_class": "MarianTokenizer"}
|
vocab.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|