Q-bert commited on
Commit
a01d92a
·
1 Parent(s): fcba8f0

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +4 -4
tokenizer_config.json CHANGED
@@ -2,7 +2,7 @@
2
  "add_prefix_space": false,
3
  "added_tokens_decoder": {
4
  "0": {
5
- "content": "<|endoftext|>",
6
  "lstrip": false,
7
  "normalized": false,
8
  "rstrip": false,
@@ -11,10 +11,10 @@
11
  }
12
  },
13
  "additional_special_tokens": [],
14
- "bos_token": "<|endoftext|>",
15
  "clean_up_tokenization_spaces": true,
16
- "eos_token": "<|endoftext|>",
17
  "model_max_length": 1024,
18
  "tokenizer_class": "GPT2Tokenizer",
19
- "unk_token": "<|endoftext|>"
20
  }
 
2
  "add_prefix_space": false,
3
  "added_tokens_decoder": {
4
  "0": {
5
+ "content": "<|endofmolecule|>",
6
  "lstrip": false,
7
  "normalized": false,
8
  "rstrip": false,
 
11
  }
12
  },
13
  "additional_special_tokens": [],
14
+ "bos_token": "<|startofmolecule|>",
15
  "clean_up_tokenization_spaces": true,
16
+ "eos_token": "<|endofmolecule|>",
17
  "model_max_length": 1024,
18
  "tokenizer_class": "GPT2Tokenizer",
19
+ "unk_token": "<|UNK|>"
20
  }