Update tokenization_kimi.py
Browse files- tokenization_kimi.py +1 -2
tokenization_kimi.py
CHANGED
|
@@ -176,7 +176,6 @@ class TikTokenTokenizer(PreTrainedTokenizer):
|
|
| 176 |
# NOTE: our encode method is not compatible with the super().encode method,
|
| 177 |
# e.g. split_special_tokens' default is True in our encode method.
|
| 178 |
if len(kwargs) > 0:
|
| 179 |
-
logger.warning( f"Calling super().encode with {kwargs}" )
|
| 180 |
return super().encode(text, **kwargs)
|
| 181 |
|
| 182 |
assert type(text) is str
|
|
@@ -344,4 +343,4 @@ def deep_sort_dict(obj: Any) -> Any:
|
|
| 344 |
return {k: deep_sort_dict(v) for k, v in sorted(obj.items())}
|
| 345 |
if isinstance(obj, list):
|
| 346 |
return [deep_sort_dict(item) for item in obj]
|
| 347 |
-
return obj
|
|
|
|
| 176 |
# NOTE: our encode method is not compatible with the super().encode method,
|
| 177 |
# e.g. split_special_tokens' default is True in our encode method.
|
| 178 |
if len(kwargs) > 0:
|
|
|
|
| 179 |
return super().encode(text, **kwargs)
|
| 180 |
|
| 181 |
assert type(text) is str
|
|
|
|
| 343 |
return {k: deep_sort_dict(v) for k, v in sorted(obj.items())}
|
| 344 |
if isinstance(obj, list):
|
| 345 |
return [deep_sort_dict(item) for item in obj]
|
| 346 |
+
return obj
|