kadasterdst
/

t5-tokenizer

Wimflorijn commited on Jan 22, 2023

Commit

919fa26

1 Parent(s): fc8a5a4

Upload tokenizer

Files changed (2) hide show

tokenizer.json CHANGED Viewed

The diff for this file is too large to render. See raw diff

tokenizer_config.json CHANGED Viewed

@@ -104,9 +104,10 @@
   "eos_token": "</s>",
   "extra_ids": 100,
   "model_max_length": 1000000000000000019884624838656,
-  "name_or_path": "C:\\PycharmProjects\\datascience-vraag-antwoord-systeem\\model\\weights",
   "pad_token": "<pad>",
-  "special_tokens_map_file": "/home/wim/Documents/datascience-vraag-antwoord-systeem/model/weights/special_tokens_map.json",
   "tokenizer_class": "T5Tokenizer",
-  "unk_token": "<unk>"
 }

   "eos_token": "</s>",
   "extra_ids": 100,
   "model_max_length": 1000000000000000019884624838656,
+  "name_or_path": "/home/wim/Documents/datascience-vraag-antwoord-systeem/model/weights",
   "pad_token": "<pad>",
+  "special_tokens_map_file": "weights/special_tokens_map.json",
   "tokenizer_class": "T5Tokenizer",
+  "unk_token": "<unk>",
+  "use_fast": true
 }