saracandu commited on
Commit
d0cd011
·
verified ·
1 Parent(s): 0013017

try to fix added_tokens error

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +2 -1
tokenizer_config.json CHANGED
@@ -7,5 +7,6 @@
7
  "bos_token": "/s",
8
  "eos_token": "s",
9
  "unk_token": "unk",
10
- "pad_token": "pad"
 
11
  }
 
7
  "bos_token": "/s",
8
  "eos_token": "s",
9
  "unk_token": "unk",
10
+ "pad_token": "pad",
11
+ "added_tokens": []
12
  }