Hugging Face's logo

Synthyra
/
ESMplusplus_small

Fill-Mask
Transformers
Safetensors
ESMplusplus
custom_code
Model card Files
xet
 Community
4
lhallee commited on
Commit
3b0d74a
·
verified ·
1 Parent(s): 0c0b9c5

Upload tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer.json +18 -0
tokenizer.json CHANGED
@@ -83,17 +83,35 @@
83
  }
84
  ],
85
  "pair": [
 
 
 
 
 
 
86
  {
87
  "Sequence": {
88
  "id": "A",
89
  "type_id": 0
90
  }
91
  },
 
 
 
 
 
 
92
  {
93
  "Sequence": {
94
  "id": "B",
95
  "type_id": 1
96
  }
 
 
 
 
 
 
97
  }
98
  ],
99
  "special_tokens": {
 
83
  }
84
  ],
85
  "pair": [
86
+ {
87
+ "SpecialToken": {
88
+ "id": "<cls>",
89
+ "type_id": 0
90
+ }
91
+ },
92
  {
93
  "Sequence": {
94
  "id": "A",
95
  "type_id": 0
96
  }
97
  },
98
+ {
99
+ "SpecialToken": {
100
+ "id": "<eos>",
101
+ "type_id": 0
102
+ }
103
+ },
104
  {
105
  "Sequence": {
106
  "id": "B",
107
  "type_id": 1
108
  }
109
+ },
110
+ {
111
+ "SpecialToken": {
112
+ "id": "<eos>",
113
+ "type_id": 1
114
+ }
115
  }
116
  ],
117
  "special_tokens": {