Upload folder using huggingface_hub
Browse files
README.md
CHANGED
|
@@ -20,7 +20,7 @@ Bilingual (Chinese / English) SentencePiece **Unigram** tokenizer with a
|
|
| 20 |
```python
|
| 21 |
from transformers import AutoTokenizer
|
| 22 |
|
| 23 |
-
tok = AutoTokenizer.from_pretrained("
|
| 24 |
ids = tok.encode("深度学习是机器学习的一个分支。")
|
| 25 |
# BOS (<s>, id=2) is prepended automatically (add_bos_token=True).
|
| 26 |
```
|
|
|
|
| 20 |
```python
|
| 21 |
from transformers import AutoTokenizer
|
| 22 |
|
| 23 |
+
tok = AutoTokenizer.from_pretrained("carry121/k3-32k-tokenizer")
|
| 24 |
ids = tok.encode("深度学习是机器学习的一个分支。")
|
| 25 |
# BOS (<s>, id=2) is prepended automatically (add_bos_token=True).
|
| 26 |
```
|