Create tokenization_hy.py
Browse files- tokenization_hy.py +1 -1
tokenization_hy.py
CHANGED
|
@@ -295,4 +295,4 @@ if __name__ == "__main__":
|
|
| 295 |
print(ids)
|
| 296 |
text2 = tokenizer.convert_tokens_to_string(tokens)
|
| 297 |
print(text2)
|
| 298 |
-
ids2 = tokenizer.convert_tokens_to_ids(tokens)
|
|
|
|
| 295 |
print(ids)
|
| 296 |
text2 = tokenizer.convert_tokens_to_string(tokens)
|
| 297 |
print(text2)
|
| 298 |
+
ids2 = tokenizer.convert_tokens_to_ids(tokens)
|