Update BertForMorphTagging.py
Browse files- BertForMorphTagging.py +2 -0
BertForMorphTagging.py
CHANGED
|
@@ -177,6 +177,8 @@ def parse_logits(input_ids: List[List[int]], sentences: List[str], tokenizer: Be
|
|
| 177 |
# { pos: str, feats: dict, prefixes: List[str], suffix: str | bool, suffix_feats: dict | None}
|
| 178 |
special_toks = tokenizer.all_special_tokens
|
| 179 |
special_toks.remove(tokenizer.unk_token)
|
|
|
|
|
|
|
| 180 |
ret = []
|
| 181 |
for sent_idx,sentence in enumerate(sentences):
|
| 182 |
input_id_strs = tokenizer.convert_ids_to_tokens(input_ids[sent_idx])
|
|
|
|
| 177 |
# { pos: str, feats: dict, prefixes: List[str], suffix: str | bool, suffix_feats: dict | None}
|
| 178 |
special_toks = tokenizer.all_special_tokens
|
| 179 |
special_toks.remove(tokenizer.unk_token)
|
| 180 |
+
special_toks.remove(tokenizer.mask_token)
|
| 181 |
+
|
| 182 |
ret = []
|
| 183 |
for sent_idx,sentence in enumerate(sentences):
|
| 184 |
input_id_strs = tokenizer.convert_ids_to_tokens(input_ids[sent_idx])
|