Kevin Hamon commited on
Commit ·
edfeb50
1
Parent(s): 1b49565
debug
Browse files- tokenizer.py +5 -0
tokenizer.py
CHANGED
|
@@ -71,6 +71,11 @@ class ChessTokenizer(PreTrainedTokenizer):
|
|
| 71 |
kwargs.pop("unk_token", None)
|
| 72 |
kwargs.pop("sep_token", None)
|
| 73 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 74 |
# Load or create vocabulary
|
| 75 |
if vocab is not None:
|
| 76 |
self._vocab = vocab
|
|
|
|
| 71 |
kwargs.pop("unk_token", None)
|
| 72 |
kwargs.pop("sep_token", None)
|
| 73 |
|
| 74 |
+
print("Initializing ChessTokenizer")
|
| 75 |
+
print(f" vocab_file: {vocab_file}")
|
| 76 |
+
print(f" vocab provided: {vocab is not None}")
|
| 77 |
+
print(f" vocab: {vocab}")
|
| 78 |
+
|
| 79 |
# Load or create vocabulary
|
| 80 |
if vocab is not None:
|
| 81 |
self._vocab = vocab
|