Upload tokenizer.py with huggingface_hub
Browse files- tokenizer.py +1 -1
tokenizer.py
CHANGED
|
@@ -70,7 +70,7 @@ class CharacterTokenizer(PreTrainedTokenizer):
|
|
| 70 |
json.dump({
|
| 71 |
"tokenizer_class": "CharacterTokenizer",
|
| 72 |
"auto_map": {
|
| 73 |
-
"AutoTokenizer": ["tokenizer.CharacterTokenizer"]
|
| 74 |
},
|
| 75 |
"bos_token": self.bos_token,
|
| 76 |
"eos_token": self.eos_token,
|
|
|
|
| 70 |
json.dump({
|
| 71 |
"tokenizer_class": "CharacterTokenizer",
|
| 72 |
"auto_map": {
|
| 73 |
+
"AutoTokenizer": ["tokenizer.CharacterTokenizer", None]
|
| 74 |
},
|
| 75 |
"bos_token": self.bos_token,
|
| 76 |
"eos_token": self.eos_token,
|