3ed0k4 commited on
Commit
81e8963
·
verified ·
1 Parent(s): 4a030e4

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +4 -1
tokenizer_config.json CHANGED
@@ -7,5 +7,8 @@
7
  "unk_token": "<unk>",
8
  "bos_token": "<s>",
9
  "eos_token": "</s>"
10
- }
 
 
 
11
  }
 
7
  "unk_token": "<unk>",
8
  "bos_token": "<s>",
9
  "eos_token": "</s>"
10
+ },
11
+ "model_file": "krd.model",
12
+ "vocab_file": "krd.vocab",
13
+ "clean_up_tokenization_spaces": true
14
  }