kadasterdst
/

t5-tokenizer

Model card Files Files and versions

Wimflorijn commited on Apr 27, 2023

Commit

a885319

·

1 Parent(s): 9a93376

Upload tokenizer

Files changed (1) hide show

tokenizer_config.json +2 -1

tokenizer_config.json CHANGED Viewed

@@ -108,5 +108,6 @@
   "pad_token": "<pad>",
   "special_tokens_map_file": "/home/wim/Documents/datascience-vraag-antwoord-systeem/weights/special_tokens_map.json",
   "tokenizer_class": "T5Tokenizer",
-  "unk_token": "<unk>"
 }

   "pad_token": "<pad>",
   "special_tokens_map_file": "/home/wim/Documents/datascience-vraag-antwoord-systeem/weights/special_tokens_map.json",
   "tokenizer_class": "T5Tokenizer",
+  "unk_token": "<unk>",
+  "use_fast": true
 }