marij868 commited on
Commit
126d64d
·
1 Parent(s): 67767ab

Upload tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -1
tokenizer_config.json CHANGED
@@ -103,6 +103,7 @@
103
  "<unk_101>",
104
  "<unk_102>"
105
  ],
 
106
  "eos_token": "</s>",
107
  "full_tokenizer_file": null,
108
  "mask_token": "<mask_2>",
@@ -111,7 +112,6 @@
111
  "offset": 103,
112
  "pad_token": "<pad>",
113
  "sp_model_kwargs": {},
114
- "special_tokens_map_file": null,
115
  "tokenizer_class": "PegasusTokenizer",
116
  "unk_token": "<unk>"
117
  }
 
103
  "<unk_101>",
104
  "<unk_102>"
105
  ],
106
+ "clean_up_tokenization_spaces": true,
107
  "eos_token": "</s>",
108
  "full_tokenizer_file": null,
109
  "mask_token": "<mask_2>",
 
112
  "offset": 103,
113
  "pad_token": "<pad>",
114
  "sp_model_kwargs": {},
 
115
  "tokenizer_class": "PegasusTokenizer",
116
  "unk_token": "<unk>"
117
  }