Asaf Yehoodai
commited on
Commit
·
f9d2efc
1
Parent(s):
4f45b27
add tokenizer
Browse files- added_tokens.json +1 -0
- source.spm +0 -0
- special_tokens_map.json +1 -0
- target.spm +0 -0
- tokenizer_config.json +1 -0
- vocab.json +0 -0
added_tokens.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"<ENT0>": 58101, "</ENT0>": 58102, "<ENT1>": 58103, "</ENT1>": 58104, "<ENT2>": 58105, "</ENT2>": 58106, "<ENT3>": 58107, "</ENT3>": 58108, "<ENT4>": 58109, "</ENT4>": 58110, "<ENT5>": 58111, "</ENT5>": 58112, "<ENT6>": 58113, "</ENT6>": 58114, "<ENT7>": 58115, "</ENT7>": 58116, "<ENT8>": 58117, "</ENT8>": 58118, "<ENT9>": 58119, "</ENT9>": 58120, "<ENT10>": 58121, "</ENT10>": 58122, "<ENT11>": 58123, "</ENT11>": 58124, "<ENT12>": 58125, "</ENT12>": 58126, "<ENT13>": 58127, "</ENT13>": 58128, "<ENT14>": 58129, "</ENT14>": 58130, "<ENT15>": 58131, "</ENT15>": 58132, "<ENT16>": 58133, "</ENT16>": 58134, "<ENT17>": 58135, "</ENT17>": 58136, "<ENT18>": 58137, "</ENT18>": 58138, "<ENT19>": 58139, "</ENT19>": 58140, "<MALE>": 58141, "</MALE>": 58142, "<FEMALE>": 58143, "</FEMALE>": 58144}
|
source.spm
ADDED
|
Binary file (768 kB). View file
|
|
|
special_tokens_map.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>"}
|
target.spm
ADDED
|
Binary file (797 kB). View file
|
|
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"source_lang": "en", "target_lang": "de", "unk_token": "<unk>", "eos_token": "</s>", "pad_token": "<pad>", "model_max_length": 512, "sp_model_kwargs": {}, "special_tokens_map_file": null, "tokenizer_file": null, "name_or_path": "/cs/labs/oabend/asaf305612939/co-ref-project/mt-output/paper_models/de/en_de_coref_exp", "tokenizer_class": "MarianTokenizer"}
|
vocab.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|