Update README.md
Browse files
README.md
CHANGED
|
@@ -7,7 +7,7 @@ tags:
|
|
| 7 |
license: apache-2.0
|
| 8 |
---
|
| 9 |
|
| 10 |
-
#
|
| 11 |
|
| 12 |
This chess tokenizer uses a large vocabulary (~844 tokens) with semantically meaningful units like 'w.', 'b.', piece+square combinations ('♙e4', '♞f6'), and complete suffixes ('..', '.x.', '.+').
|
| 13 |
|
|
@@ -19,7 +19,7 @@ This design reduces sequence length by ~60% compared to character-level tokeniza
|
|
| 19 |
from transformers import AutoTokenizer
|
| 20 |
|
| 21 |
# Load tokenizer directly from HuggingFace
|
| 22 |
-
tokenizer = AutoTokenizer.from_pretrained("ankanmbz/
|
| 23 |
|
| 24 |
# Tokenize chess moves
|
| 25 |
text = "w.♙e2♙e4.."
|
|
|
|
| 7 |
license: apache-2.0
|
| 8 |
---
|
| 9 |
|
| 10 |
+
# chess-Tok: Efficient Chess Move Tokenizer
|
| 11 |
|
| 12 |
This chess tokenizer uses a large vocabulary (~844 tokens) with semantically meaningful units like 'w.', 'b.', piece+square combinations ('♙e4', '♞f6'), and complete suffixes ('..', '.x.', '.+').
|
| 13 |
|
|
|
|
| 19 |
from transformers import AutoTokenizer
|
| 20 |
|
| 21 |
# Load tokenizer directly from HuggingFace
|
| 22 |
+
tokenizer = AutoTokenizer.from_pretrained("ankanmbz/chess-tok", trust_remote_code=True)
|
| 23 |
|
| 24 |
# Tokenize chess moves
|
| 25 |
text = "w.♙e2♙e4.."
|