eyad-silx commited on
Commit
8736db6
·
verified ·
1 Parent(s): 66b478a

Update tokenization_kimi.py

Browse files
Files changed (1) hide show
  1. tokenization_kimi.py +1 -2
tokenization_kimi.py CHANGED
@@ -176,7 +176,6 @@ class TikTokenTokenizer(PreTrainedTokenizer):
176
  # NOTE: our encode method is not compatible with the super().encode method,
177
  # e.g. split_special_tokens' default is True in our encode method.
178
  if len(kwargs) > 0:
179
- logger.warning( f"Calling super().encode with {kwargs}" )
180
  return super().encode(text, **kwargs)
181
 
182
  assert type(text) is str
@@ -344,4 +343,4 @@ def deep_sort_dict(obj: Any) -> Any:
344
  return {k: deep_sort_dict(v) for k, v in sorted(obj.items())}
345
  if isinstance(obj, list):
346
  return [deep_sort_dict(item) for item in obj]
347
- return obj
 
176
  # NOTE: our encode method is not compatible with the super().encode method,
177
  # e.g. split_special_tokens' default is True in our encode method.
178
  if len(kwargs) > 0:
 
179
  return super().encode(text, **kwargs)
180
 
181
  assert type(text) is str
 
343
  return {k: deep_sort_dict(v) for k, v in sorted(obj.items())}
344
  if isinstance(obj, list):
345
  return [deep_sort_dict(item) for item in obj]
346
+ return obj