xezpeleta commited on
Commit
bfc3ef9
·
verified ·
1 Parent(s): 602dd93

Upload tokenizer_config.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +3 -2
tokenizer_config.json CHANGED
@@ -35,5 +35,6 @@
35
  "sp_model_kwargs": {},
36
  "target_lang": "modelEuskeraConvertir",
37
  "tokenizer_class": "MarianTokenizer",
38
- "unk_token": "<unk>"
39
- }
 
 
35
  "sp_model_kwargs": {},
36
  "target_lang": "modelEuskeraConvertir",
37
  "tokenizer_class": "MarianTokenizer",
38
+ "unk_token": "<unk>",
39
+ "source_spm": "source.spm"
40
+ }