Hugging Face's logo

Tamek
/
Mirai_Model

Safetensors
llama
Model card Files
xet
 Community
Tamek commited on
Commit
fd33fbb
·
verified ·
1 Parent(s): b097c5a

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +13 -13
tokenizer_config.json CHANGED
@@ -1,13 +1,13 @@
1
- {
2
- "backend": "tokenizers",
3
- "bos_token": "<|begin_of_text|>",
4
- "clean_up_tokenization_spaces": true,
5
- "eos_token": "<|eot_id|>",
6
- "is_local": false,
7
- "model_input_names": [
8
- "input_ids",
9
- "attention_mask"
10
- ],
11
- "model_max_length": 1000000000000000019884624838656,
12
- "tokenizer_class": "TokenizersBackend"
13
- }
 
1
+ {
2
+ "backend": "tokenizers",
3
+ "bos_token": "<|begin_of_text|>",
4
+ "clean_up_tokenization_spaces": true,
5
+ "eos_token": "<|eot_id|>",
6
+ "is_local": false,
7
+ "model_input_names": [
8
+ "input_ids",
9
+ "attention_mask"
10
+ ],
11
+ "model_max_length": 1000000000000000019884624838656,
12
+ "tokenizer_class": "PreTrainedTokenizerFast"
13
+ }