Q-bert
/

SMILETokenizer

Model card Files Files and versions

Q-bert commited on Oct 7, 2023

Commit

a50b560

·

1 Parent(s): 954f705

Update tokenizer.json

Files changed (1) hide show

tokenizer.json +2 -2

tokenizer.json CHANGED Viewed

@@ -5,7 +5,7 @@
   "added_tokens": [
     {
       "id": 0,
-      "content": "<|endoftext|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -41,7 +41,7 @@
     "fuse_unk": false,
     "byte_fallback": false,
     "vocab": {
-      "<|endoftext|>": 0,
       "!": 1,
       "\"": 2,
       "#": 3,

   "added_tokens": [
     {
       "id": 0,
+      "content": "<|endofmolecule|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
     "fuse_unk": false,
     "byte_fallback": false,
     "vocab": {
+      "<|endofmolecule|>": 0,
       "!": 1,
       "\"": 2,
       "#": 3,