ppbrown commited on
Commit
62c4a4d
·
verified ·
1 Parent(s): d1c054c

Upload tokenizer/tokenizer_config.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. tokenizer/tokenizer_config.json +2 -2
tokenizer/tokenizer_config.json CHANGED
@@ -19,12 +19,12 @@
19
  }
20
  },
21
  "bos_token": "<|startoftext|>",
22
- "clean_up_tokenization_spaces": false,
23
  "do_lower_case": true,
24
  "eos_token": "<|endoftext|>",
25
  "errors": "replace",
26
  "extra_special_tokens": {},
27
- "model_max_length": 77,
28
  "pad_token": "<|endoftext|>",
29
  "tokenizer_class": "CLIPTokenizer",
30
  "unk_token": "<|endoftext|>"
 
19
  }
20
  },
21
  "bos_token": "<|startoftext|>",
22
+ "clean_up_tokenization_spaces": true,
23
  "do_lower_case": true,
24
  "eos_token": "<|endoftext|>",
25
  "errors": "replace",
26
  "extra_special_tokens": {},
27
+ "model_max_length": 248,
28
  "pad_token": "<|endoftext|>",
29
  "tokenizer_class": "CLIPTokenizer",
30
  "unk_token": "<|endoftext|>"