Spaces:
Paused
Paused
Update app.py
Browse files
app.py
CHANGED
|
@@ -26,9 +26,9 @@ tokenizer.train_from_iterator(
|
|
| 26 |
special_tokens=["<s>", "<pad>", "</s>", "<unk>", "<mask>", "<|user|>", "<|bot|>", "<|end|>"]
|
| 27 |
)
|
| 28 |
|
| 29 |
-
tokenizer.save("custom_tokenizer.json")
|
| 30 |
|
| 31 |
-
tokenizer = PreTrainedTokenizerFast(tokenizer_file="custom_tokenizer.json")
|
| 32 |
|
| 33 |
tokenizer.bos_token = "<s>"
|
| 34 |
tokenizer.eos_token = "</s>"
|
|
|
|
| 26 |
special_tokens=["<s>", "<pad>", "</s>", "<unk>", "<mask>", "<|user|>", "<|bot|>", "<|end|>"]
|
| 27 |
)
|
| 28 |
|
| 29 |
+
tokenizer.save("/tmp/custom_tokenizer.json")
|
| 30 |
|
| 31 |
+
tokenizer = PreTrainedTokenizerFast(tokenizer_file="/tmp/custom_tokenizer.json")
|
| 32 |
|
| 33 |
tokenizer.bos_token = "<s>"
|
| 34 |
tokenizer.eos_token = "</s>"
|