Update README.md
Browse files
README.md
CHANGED
|
@@ -55,7 +55,7 @@ Evaluated across varied modalities, the vocabulary yields a highly competitive *
|
|
| 55 |
from transformers import AutoTokenizer
|
| 56 |
|
| 57 |
# Initialize the Quark2Tokenizer
|
| 58 |
-
tokenizer = AutoTokenizer.from_pretrained("ThingAI/
|
| 59 |
|
| 60 |
sequence = "<|thinking|>\nExecuting field equations...\n<|/thinking|>G_{\mu\nu} = \\frac{8\\pi G}{c^4} T_{\mu\nu}"
|
| 61 |
tokens = tokenizer.encode(sequence)
|
|
|
|
| 55 |
from transformers import AutoTokenizer
|
| 56 |
|
| 57 |
# Initialize the Quark2Tokenizer
|
| 58 |
+
tokenizer = AutoTokenizer.from_pretrained("ThingAI/Quark2Tokenizer")
|
| 59 |
|
| 60 |
sequence = "<|thinking|>\nExecuting field equations...\n<|/thinking|>G_{\mu\nu} = \\frac{8\\pi G}{c^4} T_{\mu\nu}"
|
| 61 |
tokens = tokenizer.encode(sequence)
|