courage17340 hmellor HF Staff commited on
Commit
398eede
·
verified ·
1 Parent(s): 4f50fd8

Make model compatible with Transformers v5 (#34)

Browse files

- Make model compatible with Transformers v5 (7ace13de4d2bd13c951c6c969632d5322eb1c299)


Co-authored-by: Harry Mellor <hmellor@users.noreply.huggingface.co>

Files changed (1) hide show
  1. tokenization_moonshot.py +1 -1
tokenization_moonshot.py CHANGED
@@ -17,7 +17,7 @@ from tiktoken.load import load_tiktoken_bpe
17
  from tokenizers import AddedToken
18
  from transformers.tokenization_utils import PreTrainedTokenizer
19
  from transformers.utils import to_py_obj
20
- from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode
21
 
22
 
23
  logger = getLogger(__name__)
 
17
  from tokenizers import AddedToken
18
  from transformers.tokenization_utils import PreTrainedTokenizer
19
  from transformers.utils import to_py_obj
20
+ from transformers.convert_slow_tokenizer import bytes_to_unicode
21
 
22
 
23
  logger = getLogger(__name__)