Update README.md
Browse files
README.md
CHANGED
|
@@ -4,6 +4,6 @@ BPE tokenizer based on `tiktoken.cl100k_base`.
|
|
| 4 |
|
| 5 |
```
|
| 6 |
from transformers import AutoTokenizer
|
| 7 |
-
tokenizer = AutoTokenizer.from_pretrained("
|
| 8 |
tokenizer("hello, world!", return_tensors='pt')
|
| 9 |
```
|
|
|
|
| 4 |
|
| 5 |
```
|
| 6 |
from transformers import AutoTokenizer
|
| 7 |
+
tokenizer = AutoTokenizer.from_pretrained("stabilityai/arcade100k", trust_remote_code=True)
|
| 8 |
tokenizer("hello, world!", return_tensors='pt')
|
| 9 |
```
|