runfar commited on
Commit
fa469dd
·
verified ·
1 Parent(s): 6aab4d0

Upload tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -1
tokenizer_config.json CHANGED
@@ -51349,7 +51349,7 @@
51349
  "sfr_token": "<start_function_response>",
51350
  "sp_model_kwargs": null,
51351
  "spaces_between_special_tokens": false,
51352
- "tokenizer_class": "GemmaTokenizer",
51353
  "unk_token": "<unk>",
51354
  "use_default_system_prompt": false
51355
  }
 
51349
  "sfr_token": "<start_function_response>",
51350
  "sp_model_kwargs": null,
51351
  "spaces_between_special_tokens": false,
51352
+ "tokenizer_class": "GemmaTokenizerFast",
51353
  "unk_token": "<unk>",
51354
  "use_default_system_prompt": false
51355
  }