Hugging Face's logo

aszymanska
/
bk-sdm-tiny-vpred

Text-to-Image
Model card Files
xet
Community
Anna Szymańska commited on
Commit
d198c8f
·
1 Parent(s): 5bcbf85

Update tokenizer single and pair

Browse files
Files changed (1) hide show
  1. tokenizer/tokenizer.json +5 -5
tokenizer/tokenizer.json CHANGED
@@ -48,7 +48,7 @@
48
  "single": [
49
  {
50
  "SpecialToken": {
51
- "id": "[CLS]",
52
  "type_id": 0
53
  }
54
  },
@@ -60,7 +60,7 @@
60
  },
61
  {
62
  "SpecialToken": {
63
- "id": "[SEP]",
64
  "type_id": 0
65
  }
66
  }
@@ -68,7 +68,7 @@
68
  "pair": [
69
  {
70
  "SpecialToken": {
71
- "id": "[CLS]",
72
  "type_id": 0
73
  }
74
  },
@@ -80,7 +80,7 @@
80
  },
81
  {
82
  "SpecialToken": {
83
- "id": "[SEP]",
84
  "type_id": 0
85
  }
86
  },
@@ -92,7 +92,7 @@
92
  },
93
  {
94
  "SpecialToken": {
95
- "id": "[SEP]",
96
  "type_id": 1
97
  }
98
  }
 
48
  "single": [
49
  {
50
  "SpecialToken": {
51
+ "id": "<|startoftext|>",
52
  "type_id": 0
53
  }
54
  },
 
60
  },
61
  {
62
  "SpecialToken": {
63
+ "id": "<|endoftext|>",
64
  "type_id": 0
65
  }
66
  }
 
68
  "pair": [
69
  {
70
  "SpecialToken": {
71
+ "id": "<|startoftext|>",
72
  "type_id": 0
73
  }
74
  },
 
80
  },
81
  {
82
  "SpecialToken": {
83
+ "id": "<|endoftext|>",
84
  "type_id": 0
85
  }
86
  },
 
92
  },
93
  {
94
  "SpecialToken": {
95
+ "id": "<|endoftext|>",
96
  "type_id": 1
97
  }
98
  }