Update README.md, tokenization_deberta_v2_jumanpp.py, tokenization_deberta_v2_jumanpp_fast.py

README.md

@@ -29,8 +29,8 @@ You can use this model for masked language modeling as follows:
 
 ```python
 from transformers import AutoTokenizer, AutoModelForMaskedLM
-tokenizer = AutoTokenizer.from_pretrained('ku-nlp/deberta-v2-base-japanese', trust_remote_code=True)
-model = AutoModelForMaskedLM.from_pretrained('ku-nlp/deberta-v2-base-japanese')
+tokenizer = AutoTokenizer.from_pretrained('ku-nlp/deberta-v2-base-japanese-with-auto-jumanpp', trust_remote_code=True)
+model = AutoModelForMaskedLM.from_pretrained('ku-nlp/deberta-v2-base-japanese-with-auto-jumanpp')
 
 sentence = '京都大学で自然言語処理を[MASK]する。'
 encoding = tokenizer(sentence, return_tensors='pt')
@@ -41,9 +41,8 @@ You can also fine-tune this model on downstream tasks.
 
 ## Tokenization
 
-~~The input text should be segmented into words by [Juman++](https://github.com/ku-nlp/jumanpp) in advance. [Juman++ 2.0.0-rc3](https://github.com/ku-nlp/jumanpp/releases/tag/v2.0.0-rc3) was used for pre-training. Each word is tokenized into subwords by [sentencepiece](https://github.com/google/sentencepiece).~~
-
-UPDATE: The input text is internally segmented by [Juman++](https://github.com/ku-nlp/jumanpp) within `DebertaV2JumanppTokenizer(Fast)`, so there's no need to segment it in advance. To use `DebertaV2JumanppTokenizer(Fast)`, you need to install [Juman++ 2.0.0-rc3](https://github.com/ku-nlp/jumanpp/releases/tag/v2.0.0-rc3) and [rhoknp](https://github.com/ku-nlp/rhoknp).
+The input text is internally segmented by [Juman++](https://github.com/ku-nlp/jumanpp) within `DebertaV2JumanppTokenizer` or `DebertaV2JumanppTokenizerFast`, so there's no need to segment it in advance.
+To use `DebertaV2JumanppTokenizer` or `DebertaV2JumanppTokenizerFast`, you need to install [Juman++ 2.0.0-rc3](https://github.com/ku-nlp/jumanpp/releases/tag/v2.0.0-rc3) and [rhoknp](https://github.com/ku-nlp/rhoknp).
 
 ## Training data
 

tokenization_deberta_v2_jumanpp.py

@@ -24,7 +24,7 @@ class JumanppTokenizer:
                 "You need to install rhoknp to use JumanppPreTokenizer. "
                 "See https://github.com/ku-nlp/rhoknp for installation."
             )
-        self.juman = rhoknp.Jumanpp()
+        self.jumanpp = rhoknp.Jumanpp()
 
     def tokenize(self, text: str) -> str:
-        return " ".join([morpheme.surf for morpheme in self.juman.apply_to_sentence(text).morphemes])
+        return " ".join([morpheme.surf for morpheme in self.jumanpp.apply_to_sentence(text).morphemes])

tokenization_deberta_v2_jumanpp_fast.py

@@ -1,4 +1,5 @@
 import copy
+from typing import List
 
 from tokenizers import NormalizedString, PreTokenizedString, normalizers, pre_tokenizers
 from transformers import DebertaV2TokenizerFast
@@ -54,11 +55,11 @@ class JumanppPreTokenizer:
                 "You need to install rhoknp to use JumanppPreTokenizer. "
                 "See https://github.com/ku-nlp/rhoknp for installation."
             )
-        self.juman = rhoknp.Jumanpp()
+        self.jumanpp = rhoknp.Jumanpp()
 
     def pre_tokenize(self, pretok: PreTokenizedString):
         pretok.split(self.jumanpp_split)
 
-    def jumanpp_split(self, i: int, normalized_string: NormalizedString) -> list[NormalizedString]:
-        offsets = [morpheme.span for morpheme in self.juman.apply_to_sentence(str(normalized_string)).morphemes]
+    def jumanpp_split(self, i: int, normalized_string: NormalizedString) -> List[NormalizedString]:
+        offsets = [morpheme.span for morpheme in self.jumanpp.apply_to_sentence(str(normalized_string)).morphemes]
         return [normalized_string[offset[0]:offset[1]] for offset in offsets]