Model save

README.md

@@ -21,28 +21,28 @@ model-index:
       split: validation
     metrics:
     - type: f1
-      value: 0.9352911058579663
+      value: 0.9270548177755096
       name: Null F1
     - type: f1
-      value: 0.8223238635196441
+      value: 0.8339235583777782
       name: Lemma F1
     - type: f1
-      value: 0.7874293202680182
+      value: 0.7885002678867238
       name: Morphology F1
     - type: accuracy
-      value: 0.7509689490800553
+      value: 0.7653227685854114
       name: Ud Jaccard
     - type: accuracy
-      value: 0.7934583515045791
+      value: 0.7962406996475656
       name: Eud Jaccard
     - type: f1
-      value: 0.5310531282679114
+      value: 0.6438483915854029
       name: Miscs F1
     - type: f1
-      value: 0.6223423025329784
+      value: 0.6179291073868571
       name: Deepslot F1
     - type: f1
-      value: 0.6145897578961568
+      value: 0.6220501826358034
       name: Semclass F1
 ---
 

config.json

@@ -25,7 +25,7 @@
   "null_classifier_hidden_size": 512,
   "semclass_classifier_hidden_size": 512,
   "torch_dtype": "float32",
-  "transformers_version": "4.51.3",
+  "transformers_version": "4.52.3",
   "vocabulary": {
     "deepslot": {
       "0": "$Dislocation",
@@ -370,213 +370,213 @@
       "1": "ADJ#Adjective#Degree=Cmp",
       "2": "ADJ#Adjective#Degree=Pos",
       "3": "ADJ#Adjective#Degree=Sup",
-      "4": "ADJ#None#Degree=Cmp",
-      "5": "ADJ#None#Degree=Pos",
-      "6": "ADJ#None#Degree=Pos|NumType=Ord",
-      "7": "ADJ#None#Degree=Sup",
-      "8": "ADJ#None#None",
-      "9": "ADJ#Numeral#Degree=Pos|NumForm=Digit|NumType=Ord",
-      "10": "ADJ#Numeral#Degree=Pos|NumForm=Word|NumType=Ord",
-      "11": "ADJ#Prefixoid#None",
-      "12": "ADP#Adverb#None",
-      "13": "ADP#None#None",
-      "14": "ADP#Preposition#None",
+      "4": "ADJ#Numeral#Degree=Pos|NumForm=Digit|NumType=Ord",
+      "5": "ADJ#Numeral#Degree=Pos|NumForm=Word|NumType=Ord",
+      "6": "ADJ#Prefixoid#_",
+      "7": "ADJ#_#Degree=Cmp",
+      "8": "ADJ#_#Degree=Pos",
+      "9": "ADJ#_#Degree=Pos|NumType=Ord",
+      "10": "ADJ#_#Degree=Sup",
+      "11": "ADJ#_#_",
+      "12": "ADP#Adverb#_",
+      "13": "ADP#Preposition#_",
+      "14": "ADP#_#_",
       "15": "ADV#Adjective#Degree=Pos",
       "16": "ADV#Adverb#Degree=Cmp",
       "17": "ADV#Adverb#Degree=Pos",
       "18": "ADV#Adverb#Degree=Pos|NumType=Mult",
       "19": "ADV#Adverb#Degree=Sup",
-      "20": "ADV#Adverb#None",
-      "21": "ADV#Adverb#NumType=Mult",
-      "22": "ADV#Adverb#Polarity=Neg",
-      "23": "ADV#Adverb#PronType=Dem",
+      "20": "ADV#Adverb#NumType=Mult",
+      "21": "ADV#Adverb#Polarity=Neg",
+      "22": "ADV#Adverb#PronType=Dem",
+      "23": "ADV#Adverb#_",
       "24": "ADV#Invariable#Degree=Cmp",
-      "25": "ADV#Invariable#None",
-      "26": "ADV#None#Degree=Cmp",
-      "27": "ADV#None#Degree=Pos",
-      "28": "ADV#None#Degree=Sup",
-      "29": "ADV#None#None",
-      "30": "ADV#None#NumType=Mult",
-      "31": "ADV#None#PronType=Dem",
-      "32": "ADV#None#PronType=Int",
-      "33": "ADV#Prefixoid#None",
-      "34": "AUX#None#Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin",
-      "35": "AUX#Verb#Mood=Ind|Number=Plur|Person=1|Tense=Past|VerbForm=Fin",
-      "36": "AUX#Verb#Mood=Ind|Number=Plur|Person=1|Tense=Pres|VerbForm=Fin",
-      "37": "AUX#Verb#Mood=Ind|Number=Plur|Person=2|Tense=Pres|VerbForm=Fin",
-      "38": "AUX#Verb#Mood=Ind|Number=Plur|Person=3|Tense=Past|VerbForm=Fin",
-      "39": "AUX#Verb#Mood=Ind|Number=Plur|Person=3|Tense=Pres|VerbForm=Fin",
-      "40": "AUX#Verb#Mood=Ind|Number=Sing|Person=1|Tense=Past|VerbForm=Fin",
-      "41": "AUX#Verb#Mood=Ind|Number=Sing|Person=1|Tense=Pres|VerbForm=Fin",
-      "42": "AUX#Verb#Mood=Ind|Number=Sing|Person=2|Tense=Past|VerbForm=Fin",
-      "43": "AUX#Verb#Mood=Ind|Number=Sing|Person=2|Tense=Pres|VerbForm=Fin",
-      "44": "AUX#Verb#Mood=Ind|Number=Sing|Person=3|Tense=Past|VerbForm=Fin",
-      "45": "AUX#Verb#Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin",
-      "46": "AUX#Verb#Mood=Sub|Number=Plur|Person=1|Tense=Past|VerbForm=Fin",
-      "47": "AUX#Verb#Mood=Sub|Number=Plur|Tense=Past|VerbForm=Part",
-      "48": "AUX#Verb#Number=Plur|Tense=Past|VerbForm=Part",
-      "49": "AUX#Verb#Number=Plur|Tense=Pres|VerbForm=Part",
-      "50": "AUX#Verb#VerbForm=Fin",
-      "51": "AUX#Verb#VerbForm=Ger",
-      "52": "AUX#Verb#VerbForm=Inf",
-      "53": "CCONJ#Conjunction#None",
-      "54": "CCONJ#None#None",
+      "25": "ADV#Invariable#_",
+      "26": "ADV#Prefixoid#_",
+      "27": "ADV#_#Degree=Cmp",
+      "28": "ADV#_#Degree=Pos",
+      "29": "ADV#_#Degree=Sup",
+      "30": "ADV#_#NumType=Mult",
+      "31": "ADV#_#PronType=Dem",
+      "32": "ADV#_#PronType=Int",
+      "33": "ADV#_#_",
+      "34": "AUX#Verb#Mood=Ind|Number=Plur|Person=1|Tense=Past|VerbForm=Fin",
+      "35": "AUX#Verb#Mood=Ind|Number=Plur|Person=1|Tense=Pres|VerbForm=Fin",
+      "36": "AUX#Verb#Mood=Ind|Number=Plur|Person=2|Tense=Pres|VerbForm=Fin",
+      "37": "AUX#Verb#Mood=Ind|Number=Plur|Person=3|Tense=Past|VerbForm=Fin",
+      "38": "AUX#Verb#Mood=Ind|Number=Plur|Person=3|Tense=Pres|VerbForm=Fin",
+      "39": "AUX#Verb#Mood=Ind|Number=Sing|Person=1|Tense=Past|VerbForm=Fin",
+      "40": "AUX#Verb#Mood=Ind|Number=Sing|Person=1|Tense=Pres|VerbForm=Fin",
+      "41": "AUX#Verb#Mood=Ind|Number=Sing|Person=2|Tense=Past|VerbForm=Fin",
+      "42": "AUX#Verb#Mood=Ind|Number=Sing|Person=2|Tense=Pres|VerbForm=Fin",
+      "43": "AUX#Verb#Mood=Ind|Number=Sing|Person=3|Tense=Past|VerbForm=Fin",
+      "44": "AUX#Verb#Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin",
+      "45": "AUX#Verb#Mood=Sub|Number=Plur|Person=1|Tense=Past|VerbForm=Fin",
+      "46": "AUX#Verb#Mood=Sub|Number=Plur|Tense=Past|VerbForm=Part",
+      "47": "AUX#Verb#Number=Plur|Tense=Past|VerbForm=Part",
+      "48": "AUX#Verb#Number=Plur|Tense=Pres|VerbForm=Part",
+      "49": "AUX#Verb#VerbForm=Fin",
+      "50": "AUX#Verb#VerbForm=Ger",
+      "51": "AUX#Verb#VerbForm=Inf",
+      "52": "AUX#_#Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin",
+      "53": "CCONJ#Conjunction#_",
+      "54": "CCONJ#_#_",
       "55": "DET#Adjective#PronType=Tot",
       "56": "DET#Article#Definite=Def|PronType=Art",
       "57": "DET#Article#Definite=Ind|PronType=Art",
       "58": "DET#Conjunction#Definite=Def|PronType=Art",
-      "59": "DET#None#Definite=Def|PronType=Art",
-      "60": "DET#None#Definite=EMPTY",
-      "61": "DET#None#Definite=Ind|PronType=Art",
-      "62": "DET#None#None",
-      "63": "DET#None#Number=Sing|PronType=Dem",
-      "64": "DET#None#PronType=Int",
-      "65": "DET#None#PronType=Neg",
-      "66": "DET#None#PronType=Rcp",
-      "67": "DET#None#PronType=Tot",
-      "68": "DET#Prefixoid#None",
-      "69": "DET#Pronoun#None",
-      "70": "DET#Pronoun#Number=Plur|PronType=Dem",
-      "71": "DET#Pronoun#Number=Sing|PronType=Dem",
-      "72": "DET#Pronoun#Polarity=Neg",
-      "73": "DET#Pronoun#PronType=Ind",
-      "74": "DET#Pronoun#PronType=Int",
-      "75": "DET#Pronoun#PronType=Rel",
-      "76": "DET#Pronoun#PronType=Tot",
-      "77": "INTJ#Interjection#None",
+      "59": "DET#Prefixoid#_",
+      "60": "DET#Pronoun#Number=Plur|PronType=Dem",
+      "61": "DET#Pronoun#Number=Sing|PronType=Dem",
+      "62": "DET#Pronoun#Polarity=Neg",
+      "63": "DET#Pronoun#PronType=Ind",
+      "64": "DET#Pronoun#PronType=Int",
+      "65": "DET#Pronoun#PronType=Rel",
+      "66": "DET#Pronoun#PronType=Tot",
+      "67": "DET#Pronoun#_",
+      "68": "DET#_#Definite=Def|PronType=Art",
+      "69": "DET#_#Definite=EMPTY",
+      "70": "DET#_#Definite=Ind|PronType=Art",
+      "71": "DET#_#Number=Sing|PronType=Dem",
+      "72": "DET#_#PronType=Int",
+      "73": "DET#_#PronType=Neg",
+      "74": "DET#_#PronType=Rcp",
+      "75": "DET#_#PronType=Tot",
+      "76": "DET#_#_",
+      "77": "INTJ#Interjection#_",
       "78": "NOUN#Adverb#Number=Sing",
-      "79": "NOUN#None#Number=Plur",
-      "80": "NOUN#None#Number=Sing",
-      "81": "NOUN#Noun#Abbr=Yes|Number=Plur",
-      "82": "NOUN#Noun#Abbr=Yes|Number=Sing",
-      "83": "NOUN#Noun#NumType=Frac|Number=Sing",
-      "84": "NOUN#Noun#Number=Plur",
-      "85": "NOUN#Noun#Number=Sing",
-      "86": "NOUN#Noun#Number=Sing|Polarity=Neg",
-      "87": "NOUN#Noun#VerbForm=Fin",
-      "88": "NOUN#Prefixoid#None",
-      "89": "NOUN#Prefixoid#Number=Sing",
-      "90": "NUM#None#Degree=Pos|NumType=Ord",
-      "91": "NUM#None#NumType=Card",
-      "92": "NUM#Noun#NumForm=Word|NumType=Card",
-      "93": "NUM#Numeral#None",
-      "94": "NUM#Numeral#NumForm=Digit|NumType=Card",
-      "95": "NUM#Numeral#NumForm=Digit|NumType=Frac",
-      "96": "NUM#Numeral#NumForm=Roman|NumType=Card",
-      "97": "NUM#Numeral#NumForm=Word|NumType=Card",
-      "98": "NUM#Numeral#NumType=Card",
-      "99": "PART#None#None",
-      "100": "PART#None#Polarity=Neg",
-      "101": "PART#Particle#None",
-      "102": "PART#Particle#Polarity=Neg",
-      "103": "PPROPN#None#Number=Plur",
-      "104": "PRON#None#Gender=Neut|Number=Sing|Person=3|Poss=Yes|PronType=Prs",
-      "105": "PRON#None#Number=Sing",
-      "106": "PRON#None#Number=Sing|PronType=Dem",
-      "107": "PRON#None#Number=Sing|PronType=Ind",
-      "108": "PRON#None#PronType=Int",
-      "109": "PRON#None#PronType=Rel",
-      "110": "PRON#Pronoun#Case=Acc|Gender=Fem|Number=Sing|Person=3|PronType=Prs",
-      "111": "PRON#Pronoun#Case=Acc|Gender=Fem|Number=Sing|Person=3|PronType=Prs|Reflex=Yes",
-      "112": "PRON#Pronoun#Case=Acc|Gender=Masc|Number=Sing|Person=3|PronType=Prs",
-      "113": "PRON#Pronoun#Case=Acc|Gender=Masc|Number=Sing|Person=3|PronType=Prs|Reflex=Yes",
-      "114": "PRON#Pronoun#Case=Acc|Gender=Neut|Number=Sing|Person=3|PronType=Prs",
-      "115": "PRON#Pronoun#Case=Acc|Gender=Neut|Number=Sing|Person=3|PronType=Prs|Reflex=Yes",
-      "116": "PRON#Pronoun#Case=Acc|Number=Plur|Person=1|PronType=Prs",
-      "117": "PRON#Pronoun#Case=Acc|Number=Plur|Person=1|PronType=Prs|Reflex=Yes",
-      "118": "PRON#Pronoun#Case=Acc|Number=Plur|Person=2|PronType=Prs",
-      "119": "PRON#Pronoun#Case=Acc|Number=Plur|Person=3|PronType=Prs",
-      "120": "PRON#Pronoun#Case=Acc|Number=Plur|Person=3|PronType=Prs|Reflex=Yes",
-      "121": "PRON#Pronoun#Case=Acc|Number=Sing|Person=1|PronType=Prs",
-      "122": "PRON#Pronoun#Case=Acc|Number=Sing|Person=2|PronType=Prs",
-      "123": "PRON#Pronoun#Case=Acc|Number=Sing|Person=2|PronType=Prs|Reflex=Yes",
-      "124": "PRON#Pronoun#Case=Gen|Gender=Fem|Number=Sing|Person=3|Poss=Yes|PronType=Prs",
-      "125": "PRON#Pronoun#Case=Gen|Gender=Masc|Number=Sing|Person=3|Poss=Yes|PronType=Prs",
-      "126": "PRON#Pronoun#Case=Gen|Gender=Neut|Number=Sing|Person=3|Poss=Yes|PronType=Prs",
-      "127": "PRON#Pronoun#Case=Gen|Number=Plur|Person=1|Poss=Yes|PronType=Prs",
-      "128": "PRON#Pronoun#Case=Gen|Number=Plur|Person=3|Poss=Yes|PronType=Prs",
-      "129": "PRON#Pronoun#Case=Gen|Number=Sing|Person=1|Poss=Yes|PronType=Prs",
-      "130": "PRON#Pronoun#Case=Gen|Number=Sing|Person=2|Poss=Yes|PronType=Prs",
-      "131": "PRON#Pronoun#Case=Nom|Gender=Fem|Number=Sing|Person=3|PronType=Prs",
-      "132": "PRON#Pronoun#Case=Nom|Gender=Masc|Number=Sing|Person=3|PronType=Prs",
-      "133": "PRON#Pronoun#Case=Nom|Gender=Masc|Number=Sing|Person=3|PronType=Prs|Reflex=Yes",
-      "134": "PRON#Pronoun#Case=Nom|Gender=Neut|Number=Sing|Person=3|PronType=Prs",
-      "135": "PRON#Pronoun#Case=Nom|Gender=Neut|Number=Sing|Person=3|PronType=Prs|Reflex=Yes",
-      "136": "PRON#Pronoun#Case=Nom|Number=Plur|Person=1|PronType=Prs",
-      "137": "PRON#Pronoun#Case=Nom|Number=Plur|Person=2|PronType=Prs",
-      "138": "PRON#Pronoun#Case=Nom|Number=Plur|Person=3|PronType=Prs",
-      "139": "PRON#Pronoun#Case=Nom|Number=Plur|Person=3|PronType=Prs|Reflex=Yes",
-      "140": "PRON#Pronoun#Case=Nom|Number=Sing|Person=1|PronType=Prs",
-      "141": "PRON#Pronoun#Case=Nom|Number=Sing|Person=2|PronType=Prs",
-      "142": "PRON#Pronoun#None",
-      "143": "PRON#Pronoun#Number=Plur",
-      "144": "PRON#Pronoun#Number=Plur|PronType=Dem",
-      "145": "PRON#Pronoun#Number=Plur|PronType=Tot",
-      "146": "PRON#Pronoun#Number=Sing",
-      "147": "PRON#Pronoun#Number=Sing|Polarity=Neg|PronType=Neg",
-      "148": "PRON#Pronoun#Number=Sing|PronType=Dem",
-      "149": "PRON#Pronoun#Number=Sing|PronType=Ind",
-      "150": "PRON#Pronoun#Number=Sing|PronType=Neg",
-      "151": "PRON#Pronoun#Number=Sing|Reflex=Yes",
-      "152": "PRON#Pronoun#PronType=Ind",
-      "153": "PRON#Pronoun#PronType=Int",
-      "154": "PRON#Pronoun#PronType=Rel",
-      "155": "PROPN#None#Abbr=Yes",
-      "156": "PROPN#None#Number=Plur",
-      "157": "PROPN#None#Number=Sing",
-      "158": "PROPN#Noun#Abbr=Yes|Number=Plur",
-      "159": "PROPN#Noun#Abbr=Yes|Number=Sing",
-      "160": "PROPN#Noun#Number=Plur",
-      "161": "PROPN#Noun#Number=Sing",
-      "162": "PROPN#Noun#Number=Sing|Polarity=Neg",
-      "163": "PROPN#Noun#PronType=Dem",
-      "164": "PROPN#Noun#VerbForm=Fin",
-      "165": "PROPN#Prefixoid#Number=Sing",
-      "166": "PUNCT#None#None",
-      "167": "PUNCT#PUNCT#None",
-      "168": "Prefixoid#Prefixoid#None",
-      "169": "SCONJ#Conjunction#None",
-      "170": "SCONJ#None#None",
-      "171": "SYM#Conjunction#None",
-      "172": "SYM#Noun#None",
-      "173": "SYM#Noun#Number=Sing",
-      "174": "VERB#None#Mood=Ind|Tense=Past|VerbForm=Fin",
-      "175": "VERB#None#Tense=Past|VerbForm=Part",
-      "176": "VERB#None#VerbForm=Ger",
-      "177": "VERB#None#VerbForm=Inf",
-      "178": "VERB#Verb#Mood=Imp|VerbForm=Inf",
-      "179": "VERB#Verb#Mood=Ind|Number=Plur|Person=1|Tense=Past|VerbForm=Fin",
-      "180": "VERB#Verb#Mood=Ind|Number=Plur|Person=1|Tense=Pres|VerbForm=Fin",
-      "181": "VERB#Verb#Mood=Ind|Number=Plur|Person=2|Tense=Pres|VerbForm=Fin",
-      "182": "VERB#Verb#Mood=Ind|Number=Plur|Person=3|Tense=Past|VerbForm=Fin",
-      "183": "VERB#Verb#Mood=Ind|Number=Plur|Person=3|Tense=Pres|VerbForm=Fin",
-      "184": "VERB#Verb#Mood=Ind|Number=Sing|Person=1|Tense=Past|VerbForm=Fin",
-      "185": "VERB#Verb#Mood=Ind|Number=Sing|Person=1|Tense=Pres|VerbForm=Fin",
-      "186": "VERB#Verb#Mood=Ind|Number=Sing|Person=2|Tense=Past|VerbForm=Fin",
-      "187": "VERB#Verb#Mood=Ind|Number=Sing|Person=2|Tense=Pres|VerbForm=Fin",
-      "188": "VERB#Verb#Mood=Ind|Number=Sing|Person=3|Polarity=Neg|Tense=Pres|VerbForm=Fin",
-      "189": "VERB#Verb#Mood=Ind|Number=Sing|Person=3|Tense=Past|VerbForm=Fin",
-      "190": "VERB#Verb#Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin",
-      "191": "VERB#Verb#Mood=Sub|Number=Plur|Person=1|Tense=Past|VerbForm=Fin",
-      "192": "VERB#Verb#Mood=Sub|Tense=Past|VerbForm=Part",
-      "193": "VERB#Verb#Mood=Sub|Tense=Past|VerbForm=Part|Voice=Pass",
-      "194": "VERB#Verb#Mood=Sub|VerbForm=Inf",
-      "195": "VERB#Verb#Person=1|Tense=Past|VerbForm=Part",
-      "196": "VERB#Verb#Person=1|Tense=Past|VerbForm=Part|Voice=Pass",
-      "197": "VERB#Verb#Person=1|Tense=Pres|VerbForm=Ger",
-      "198": "VERB#Verb#Person=1|Tense=Pres|VerbForm=Inf",
-      "199": "VERB#Verb#Person=1|Tense=Pres|VerbForm=Part",
-      "200": "VERB#Verb#Person=2|Tense=Pres|VerbForm=Inf",
-      "201": "VERB#Verb#Tense=Past|VerbForm=Part",
-      "202": "VERB#Verb#Tense=Past|VerbForm=Part|Voice=Pass",
-      "203": "VERB#Verb#Tense=Pres|VerbForm=Part",
-      "204": "VERB#Verb#VerbForm=Fin",
-      "205": "VERB#Verb#VerbForm=Ger",
-      "206": "VERB#Verb#VerbForm=Inf",
-      "207": "X#None#Foreign=Yes",
-      "208": "X#None#None",
-      "209": "X#None#Typo=Yes",
-      "210": "X#None#foreign=Yes"
+      "79": "NOUN#Noun#Abbr=Yes|Number=Plur",
+      "80": "NOUN#Noun#Abbr=Yes|Number=Sing",
+      "81": "NOUN#Noun#NumType=Frac|Number=Sing",
+      "82": "NOUN#Noun#Number=Plur",
+      "83": "NOUN#Noun#Number=Sing",
+      "84": "NOUN#Noun#Number=Sing|Polarity=Neg",
+      "85": "NOUN#Noun#VerbForm=Fin",
+      "86": "NOUN#Prefixoid#Number=Sing",
+      "87": "NOUN#Prefixoid#_",
+      "88": "NOUN#_#Number=Plur",
+      "89": "NOUN#_#Number=Sing",
+      "90": "NUM#Noun#NumForm=Word|NumType=Card",
+      "91": "NUM#Numeral#NumForm=Digit|NumType=Card",
+      "92": "NUM#Numeral#NumForm=Digit|NumType=Frac",
+      "93": "NUM#Numeral#NumForm=Roman|NumType=Card",
+      "94": "NUM#Numeral#NumForm=Word|NumType=Card",
+      "95": "NUM#Numeral#NumType=Card",
+      "96": "NUM#Numeral#_",
+      "97": "NUM#_#Degree=Pos|NumType=Ord",
+      "98": "NUM#_#NumType=Card",
+      "99": "PART#Particle#Polarity=Neg",
+      "100": "PART#Particle#_",
+      "101": "PART#_#Polarity=Neg",
+      "102": "PART#_#_",
+      "103": "PPROPN#_#Number=Plur",
+      "104": "PRON#Pronoun#Case=Acc|Gender=Fem|Number=Sing|Person=3|PronType=Prs",
+      "105": "PRON#Pronoun#Case=Acc|Gender=Fem|Number=Sing|Person=3|PronType=Prs|Reflex=Yes",
+      "106": "PRON#Pronoun#Case=Acc|Gender=Masc|Number=Sing|Person=3|PronType=Prs",
+      "107": "PRON#Pronoun#Case=Acc|Gender=Masc|Number=Sing|Person=3|PronType=Prs|Reflex=Yes",
+      "108": "PRON#Pronoun#Case=Acc|Gender=Neut|Number=Sing|Person=3|PronType=Prs",
+      "109": "PRON#Pronoun#Case=Acc|Gender=Neut|Number=Sing|Person=3|PronType=Prs|Reflex=Yes",
+      "110": "PRON#Pronoun#Case=Acc|Number=Plur|Person=1|PronType=Prs",
+      "111": "PRON#Pronoun#Case=Acc|Number=Plur|Person=1|PronType=Prs|Reflex=Yes",
+      "112": "PRON#Pronoun#Case=Acc|Number=Plur|Person=2|PronType=Prs",
+      "113": "PRON#Pronoun#Case=Acc|Number=Plur|Person=3|PronType=Prs",
+      "114": "PRON#Pronoun#Case=Acc|Number=Plur|Person=3|PronType=Prs|Reflex=Yes",
+      "115": "PRON#Pronoun#Case=Acc|Number=Sing|Person=1|PronType=Prs",
+      "116": "PRON#Pronoun#Case=Acc|Number=Sing|Person=2|PronType=Prs",
+      "117": "PRON#Pronoun#Case=Acc|Number=Sing|Person=2|PronType=Prs|Reflex=Yes",
+      "118": "PRON#Pronoun#Case=Gen|Gender=Fem|Number=Sing|Person=3|Poss=Yes|PronType=Prs",
+      "119": "PRON#Pronoun#Case=Gen|Gender=Masc|Number=Sing|Person=3|Poss=Yes|PronType=Prs",
+      "120": "PRON#Pronoun#Case=Gen|Gender=Neut|Number=Sing|Person=3|Poss=Yes|PronType=Prs",
+      "121": "PRON#Pronoun#Case=Gen|Number=Plur|Person=1|Poss=Yes|PronType=Prs",
+      "122": "PRON#Pronoun#Case=Gen|Number=Plur|Person=3|Poss=Yes|PronType=Prs",
+      "123": "PRON#Pronoun#Case=Gen|Number=Sing|Person=1|Poss=Yes|PronType=Prs",
+      "124": "PRON#Pronoun#Case=Gen|Number=Sing|Person=2|Poss=Yes|PronType=Prs",
+      "125": "PRON#Pronoun#Case=Nom|Gender=Fem|Number=Sing|Person=3|PronType=Prs",
+      "126": "PRON#Pronoun#Case=Nom|Gender=Masc|Number=Sing|Person=3|PronType=Prs",
+      "127": "PRON#Pronoun#Case=Nom|Gender=Masc|Number=Sing|Person=3|PronType=Prs|Reflex=Yes",
+      "128": "PRON#Pronoun#Case=Nom|Gender=Neut|Number=Sing|Person=3|PronType=Prs",
+      "129": "PRON#Pronoun#Case=Nom|Gender=Neut|Number=Sing|Person=3|PronType=Prs|Reflex=Yes",
+      "130": "PRON#Pronoun#Case=Nom|Number=Plur|Person=1|PronType=Prs",
+      "131": "PRON#Pronoun#Case=Nom|Number=Plur|Person=2|PronType=Prs",
+      "132": "PRON#Pronoun#Case=Nom|Number=Plur|Person=3|PronType=Prs",
+      "133": "PRON#Pronoun#Case=Nom|Number=Plur|Person=3|PronType=Prs|Reflex=Yes",
+      "134": "PRON#Pronoun#Case=Nom|Number=Sing|Person=1|PronType=Prs",
+      "135": "PRON#Pronoun#Case=Nom|Number=Sing|Person=2|PronType=Prs",
+      "136": "PRON#Pronoun#Number=Plur",
+      "137": "PRON#Pronoun#Number=Plur|PronType=Dem",
+      "138": "PRON#Pronoun#Number=Plur|PronType=Tot",
+      "139": "PRON#Pronoun#Number=Sing",
+      "140": "PRON#Pronoun#Number=Sing|Polarity=Neg|PronType=Neg",
+      "141": "PRON#Pronoun#Number=Sing|PronType=Dem",
+      "142": "PRON#Pronoun#Number=Sing|PronType=Ind",
+      "143": "PRON#Pronoun#Number=Sing|PronType=Neg",
+      "144": "PRON#Pronoun#Number=Sing|Reflex=Yes",
+      "145": "PRON#Pronoun#PronType=Ind",
+      "146": "PRON#Pronoun#PronType=Int",
+      "147": "PRON#Pronoun#PronType=Rel",
+      "148": "PRON#Pronoun#_",
+      "149": "PRON#_#Gender=Neut|Number=Sing|Person=3|Poss=Yes|PronType=Prs",
+      "150": "PRON#_#Number=Sing",
+      "151": "PRON#_#Number=Sing|PronType=Dem",
+      "152": "PRON#_#Number=Sing|PronType=Ind",
+      "153": "PRON#_#PronType=Int",
+      "154": "PRON#_#PronType=Rel",
+      "155": "PROPN#Noun#Abbr=Yes|Number=Plur",
+      "156": "PROPN#Noun#Abbr=Yes|Number=Sing",
+      "157": "PROPN#Noun#Number=Plur",
+      "158": "PROPN#Noun#Number=Sing",
+      "159": "PROPN#Noun#Number=Sing|Polarity=Neg",
+      "160": "PROPN#Noun#PronType=Dem",
+      "161": "PROPN#Noun#VerbForm=Fin",
+      "162": "PROPN#Prefixoid#Number=Sing",
+      "163": "PROPN#_#Abbr=Yes",
+      "164": "PROPN#_#Number=Plur",
+      "165": "PROPN#_#Number=Sing",
+      "166": "PUNCT#PUNCT#_",
+      "167": "PUNCT#_#_",
+      "168": "Prefixoid#Prefixoid#_",
+      "169": "SCONJ#Conjunction#_",
+      "170": "SCONJ#_#_",
+      "171": "SYM#Conjunction#_",
+      "172": "SYM#Noun#Number=Sing",
+      "173": "SYM#Noun#_",
+      "174": "VERB#Verb#Mood=Imp|VerbForm=Inf",
+      "175": "VERB#Verb#Mood=Ind|Number=Plur|Person=1|Tense=Past|VerbForm=Fin",
+      "176": "VERB#Verb#Mood=Ind|Number=Plur|Person=1|Tense=Pres|VerbForm=Fin",
+      "177": "VERB#Verb#Mood=Ind|Number=Plur|Person=2|Tense=Pres|VerbForm=Fin",
+      "178": "VERB#Verb#Mood=Ind|Number=Plur|Person=3|Tense=Past|VerbForm=Fin",
+      "179": "VERB#Verb#Mood=Ind|Number=Plur|Person=3|Tense=Pres|VerbForm=Fin",
+      "180": "VERB#Verb#Mood=Ind|Number=Sing|Person=1|Tense=Past|VerbForm=Fin",
+      "181": "VERB#Verb#Mood=Ind|Number=Sing|Person=1|Tense=Pres|VerbForm=Fin",
+      "182": "VERB#Verb#Mood=Ind|Number=Sing|Person=2|Tense=Past|VerbForm=Fin",
+      "183": "VERB#Verb#Mood=Ind|Number=Sing|Person=2|Tense=Pres|VerbForm=Fin",
+      "184": "VERB#Verb#Mood=Ind|Number=Sing|Person=3|Polarity=Neg|Tense=Pres|VerbForm=Fin",
+      "185": "VERB#Verb#Mood=Ind|Number=Sing|Person=3|Tense=Past|VerbForm=Fin",
+      "186": "VERB#Verb#Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin",
+      "187": "VERB#Verb#Mood=Sub|Number=Plur|Person=1|Tense=Past|VerbForm=Fin",
+      "188": "VERB#Verb#Mood=Sub|Tense=Past|VerbForm=Part",
+      "189": "VERB#Verb#Mood=Sub|Tense=Past|VerbForm=Part|Voice=Pass",
+      "190": "VERB#Verb#Mood=Sub|VerbForm=Inf",
+      "191": "VERB#Verb#Person=1|Tense=Past|VerbForm=Part",
+      "192": "VERB#Verb#Person=1|Tense=Past|VerbForm=Part|Voice=Pass",
+      "193": "VERB#Verb#Person=1|Tense=Pres|VerbForm=Ger",
+      "194": "VERB#Verb#Person=1|Tense=Pres|VerbForm=Inf",
+      "195": "VERB#Verb#Person=1|Tense=Pres|VerbForm=Part",
+      "196": "VERB#Verb#Person=2|Tense=Pres|VerbForm=Inf",
+      "197": "VERB#Verb#Tense=Past|VerbForm=Part",
+      "198": "VERB#Verb#Tense=Past|VerbForm=Part|Voice=Pass",
+      "199": "VERB#Verb#Tense=Pres|VerbForm=Part",
+      "200": "VERB#Verb#VerbForm=Fin",
+      "201": "VERB#Verb#VerbForm=Ger",
+      "202": "VERB#Verb#VerbForm=Inf",
+      "203": "VERB#_#Mood=Ind|Tense=Past|VerbForm=Fin",
+      "204": "VERB#_#Tense=Past|VerbForm=Part",
+      "205": "VERB#_#VerbForm=Ger",
+      "206": "VERB#_#VerbForm=Inf",
+      "207": "X#_#Foreign=Yes",
+      "208": "X#_#Typo=Yes",
+      "209": "X#_#_",
+      "210": "X#_#foreign=Yes"
     },
     "lemma_rule": {
       "0": "cut_prefix=0|cut_suffix=0|append_suffix=",

dependency_classifier.py

@@ -38,19 +38,21 @@ class DependencyHeadBase(nn.Module):
 
     def forward(
         self,
-        h_arc_head: Tensor,    # [batch_size, seq_len, hidden_size]
-        h_arc_dep: Tensor,     # ...
-        h_rel_head: Tensor,    # ...
-        h_rel_dep: Tensor,     # ...
-        gold_arcs: LongTensor, # [batch_size, seq_len, seq_len]
-        mask: BoolTensor       # [batch_size, seq_len]
+        h_arc_head: Tensor,        # [batch_size, seq_len, hidden_size]
+        h_arc_dep: Tensor,         # ...
+        h_rel_head: Tensor,        # ...
+        h_rel_dep: Tensor,         # ...
+        gold_arcs: LongTensor,     # [batch_size, seq_len, seq_len]
+        null_mask: BoolTensor,     # [batch_size, seq_len]
+        padding_mask: BoolTensor   # [batch_size, seq_len]
     ) -> dict[str, Tensor]:
-
+        
         # Score arcs.
-        # s_arc[:, i, j] = score of edge j -> i.
+        # s_arc[:, i, j] = score of edge i -> j.
         s_arc = self.arc_attention(h_arc_head, h_arc_dep)
         # Mask undesirable values (padding, nulls, etc.) with -inf.
-        replace_masked_values(s_arc, pairwise_mask(mask), replace_with=-1e8)
+        mask2d = pairwise_mask(null_mask & padding_mask)
+        replace_masked_values(s_arc, mask2d, replace_with=-1e8)
         # Score arcs' relations.
         # [batch_size, seq_len, seq_len, num_labels]
         s_rel = self.rel_attention(h_rel_head, h_rel_dep).permute(0, 2, 3, 1)
@@ -63,11 +65,11 @@ class DependencyHeadBase(nn.Module):
 
         # Predict arcs based on the scores.
         # [batch_size, seq_len, seq_len]
-        pred_arcs_3d = self.predict_arcs(s_arc, mask)
+        pred_arcs_matrix = self.predict_arcs(s_arc, null_mask, padding_mask)
         # [batch_size, seq_len, seq_len]
-        pred_rels_3d = self.predict_rels(s_rel)
+        pred_rels_matrix = self.predict_rels(s_rel)
         # [n_pred_arcs, 4]
-        preds_combined = self.combine_arcs_rels(pred_arcs_3d, pred_rels_3d)
+        preds_combined = self.combine_arcs_rels(pred_arcs_matrix, pred_rels_matrix)
         return {
             'preds': preds_combined,
             'loss': loss
@@ -91,8 +93,9 @@ class DependencyHeadBase(nn.Module):
     
     def predict_arcs(
         self,
-        s_arc: Tensor,   # [batch_size, seq_len, seq_len]
-        mask: BoolTensor # [batch_size, seq_len]
+        s_arc: Tensor,           # [batch_size, seq_len, seq_len]
+        null_mask: BoolTensor,   # [batch_size, seq_len]
+        padding_mask: BoolTensor # [batch_size, seq_len]
     ) -> LongTensor:
         """Predict arcs from scores."""
         raise NotImplementedError
@@ -127,42 +130,40 @@ class DependencyHead(DependencyHeadBase):
     @override
     def predict_arcs(
         self,
-        s_arc: Tensor,   # [batch_size, seq_len, seq_len]
-        mask: BoolTensor # [batch_size, seq_len]
+        s_arc: Tensor,           # [batch_size, seq_len, seq_len]
+        null_mask: BoolTensor,   # [batch_size, seq_len]
+        padding_mask: BoolTensor # [batch_size, seq_len, seq_len]
     ) -> Tensor:
 
         if self.training:
             # During training, use fast greedy decoding.
             # - [batch_size, seq_len]
-            pred_arcs_seq = s_arc.argmax(dim=-1)
+            pred_arcs_seq = s_arc.argmax(dim=1)
         else:
-            # During inference, diligently decode Maximum Spanning Tree.
-            pred_arcs_seq = self._mst_decode(s_arc, mask)
-            # FIXME
-            # pred_arcs_seq = s_arc.argmax(dim=-1)
+            # During inference, decode Maximum Spanning Tree.
+            pred_arcs_seq = self._mst_decode(s_arc, padding_mask)
 
         # Upscale arcs sequence of shape [batch_size, seq_len]
         # to matrix of shape [batch_size, seq_len, seq_len].
-        pred_arcs = F.one_hot(pred_arcs_seq, num_classes=pred_arcs_seq.size(1)).long()
+        pred_arcs = F.one_hot(pred_arcs_seq, num_classes=pred_arcs_seq.size(1)).long().transpose(1, 2)
+        # Apply mask one more time (even though s_arc is already masked),
+        # because argmax erases information about masked values.
+        mask2d = pairwise_mask(null_mask & padding_mask)
+        replace_masked_values(pred_arcs, mask2d, replace_with=0)
         return pred_arcs
 
     def _mst_decode(
         self,
-        s_arc: Tensor, # [batch_size, seq_len, seq_len]
-        mask: Tensor   # [batch_size, seq_len]
+        s_arc: Tensor,    # [batch_size, seq_len, seq_len]
+        padding_mask: Tensor
     ) -> tuple[Tensor, Tensor]:
-
+        
         batch_size = s_arc.size(0)
         device = s_arc.device
         s_arc = s_arc.cpu()
 
         # Convert scores to probabilities, as `decode_mst` expects non-negative values.
-        arc_probs = nn.functional.softmax(s_arc, dim=-1)
-        # Transpose arcs, because decode_mst defines 'energy' matrix as
-        #  energy[i,j] = "Score that `i` is the head of `j`",
-        # whereas
-        #  arc_probs[i,j] = "Probability that `j` is the head of `i`".
-        arc_probs = arc_probs.transpose(1, 2)
+        arc_probs = nn.functional.softmax(s_arc, dim=1)
 
         # `decode_mst` knows nothing about UD and ROOT, so we have to manually
         # zero probabilities of arcs leading to ROOT to make sure ROOT is a source node
@@ -177,11 +178,10 @@ class DependencyHead(DependencyHeadBase):
         pred_arcs = []
         for sample_idx in range(batch_size):
             energy = arc_probs[sample_idx]
-            # has_labels=False because we will decode them manually later.
-            lengths = mask[sample_idx].sum()
-            heads, _ = decode_mst(energy, lengths, has_labels=False)
+            length = padding_mask[sample_idx].sum()
+            heads = decode_mst(energy, length)
             # Some nodes may be isolated. Pick heads greedily in this case.
-            heads[heads <= 0] = s_arc[sample_idx].argmax(dim=-1)[heads <= 0]
+            heads[heads <= 0] = s_arc[sample_idx].argmax(dim=1)[heads <= 0]
             pred_arcs.append(heads)
 
         # shape: [batch_size, seq_len]
@@ -195,7 +195,7 @@ class DependencyHead(DependencyHeadBase):
         gold_arcs: LongTensor  # [n_arcs, 4]
     ) -> tuple[Tensor, Tensor]:
         batch_idxs, from_idxs, to_idxs, _ = gold_arcs.T
-        return F.cross_entropy(s_arc[batch_idxs, from_idxs], to_idxs)
+        return F.cross_entropy(s_arc[batch_idxs, :, to_idxs], from_idxs)
 
 
 class MultiDependencyHead(DependencyHeadBase):
@@ -206,8 +206,9 @@ class MultiDependencyHead(DependencyHeadBase):
     @override
     def predict_arcs(
         self,
-        s_arc: Tensor,   # [batch_size, seq_len, seq_len]
-        mask: BoolTensor # [batch_size, seq_len]
+        s_arc: Tensor,           # [batch_size, seq_len, seq_len]
+        null_mask: BoolTensor,   # [batch_size, seq_len]
+        padding_mask: BoolTensor # [batch_size, seq_len]
     ) -> Tensor:
         # Convert scores to probabilities.
         arc_probs = torch.sigmoid(s_arc)
@@ -263,8 +264,8 @@ class DependencyClassifier(nn.Module):
         embeddings: Tensor,    # [batch_size, seq_len, embedding_size]
         gold_ud: Tensor,       # [n_ud_arcs, 4]
         gold_eud: Tensor,      # [n_eud_arcs, 4]
-        mask_ud: Tensor,       # [batch_size, seq_len]
-        mask_eud: Tensor       # [batch_size, seq_len]
+        null_mask: Tensor,     # [batch_size, seq_len]
+        padding_mask: Tensor   # [batch_size, seq_len]
     ) -> dict[str, Tensor]:
 
         # - [batch_size, seq_len, hidden_size]
@@ -280,7 +281,8 @@ class DependencyClassifier(nn.Module):
             h_rel_head,
             h_rel_dep,
             gold_arcs=gold_ud,
-            mask=mask_ud
+            null_mask=null_mask,
+            padding_mask=padding_mask
         )
         output_eud = self.dependency_head_eud(
             h_arc_head,
@@ -288,7 +290,9 @@ class DependencyClassifier(nn.Module):
             h_rel_head,
             h_rel_dep,
             gold_arcs=gold_eud,
-            mask=mask_eud
+            # Ignore null mask in E-UD
+            null_mask=torch.ones_like(padding_mask),
+            padding_mask=padding_mask
         )
 
         return {

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3641289089a079abc37a0858a0a412bc6f031f0755e5b24d229c4bf92ce83976
+oid sha256:1618ac5132f5aa1c8525829b0a8ac2e7a0e38ae184cfd1bdcbe5ded4e90a63ee
 size 1141314800

modeling_parser.py

@@ -1,8 +1,6 @@
 from torch import nn
 from torch import LongTensor
 from transformers import PreTrainedModel
-from transformers.modeling_outputs import ModelOutput
-from dataclasses import dataclass
 
 from .configuration import CobaldParserConfig
 from .encoder import WordTransformerEncoder
@@ -17,23 +15,6 @@ from .utils import (
 )
 
 
-@dataclass
-class CobaldParserOutput(ModelOutput):
-    """
-    Output type for CobaldParser.
-    """
-    loss: float = None
-    words: list = None
-    counting_mask: LongTensor = None
-    lemma_rules: LongTensor = None
-    joint_feats: LongTensor = None
-    deps_ud: LongTensor = None
-    deps_eud: LongTensor = None
-    miscs: LongTensor = None
-    deepslots: LongTensor = None
-    semclasses: LongTensor = None
-
-
 class CobaldParser(PreTrainedModel):
     """Morpho-Syntax-Semantic Parser."""
 
@@ -119,8 +100,8 @@ class CobaldParser(PreTrainedModel):
         sent_ids: list[str] = None,
         texts: list[str] = None,
         inference_mode: bool = False
-    ) -> CobaldParserOutput:
-        result = {}
+    ) -> dict:
+        output = {}
 
         # Extra [CLS] token accounts for the case when #NULL is the first token in a sentence.
         words_with_cls = prepend_cls(words)
@@ -129,62 +110,62 @@ class CobaldParser(PreTrainedModel):
         embeddings_without_nulls = self.encoder(words_without_nulls)
         # Predict nulls.
         null_output = self.classifiers["null"](embeddings_without_nulls, counting_masks)
-        result["counting_mask"] = null_output['preds']
-        result["loss"] = null_output["loss"]
+        output["counting_mask"] = null_output['preds']
+        output["loss"] = null_output["loss"]
 
         # "Teacher forcing": during training, pass the original words (with gold nulls)
         # to the classification heads, so that they are trained upon correct sentences.
         if inference_mode:
             # Restore predicted nulls in the original sentences.
-            result["words"] = add_nulls(words, null_output["preds"])
+            output["words"] = add_nulls(words, null_output["preds"])
         else:
-            result["words"] = words
+            output["words"] = words
 
         # Encode words with nulls.
         # [batch_size, seq_len, embedding_size]
-        embeddings = self.encoder(result["words"])
+        embeddings = self.encoder(output["words"])
 
         # Predict lemmas and morphological features.
         if "lemma_rule" in self.classifiers:
             lemma_output = self.classifiers["lemma_rule"](embeddings, lemma_rules)
-            result["lemma_rules"] = lemma_output['preds']
-            result["loss"] += lemma_output['loss']
+            output["lemma_rules"] = lemma_output['preds']
+            output["loss"] += lemma_output['loss']
 
         if "joint_feats" in self.classifiers:
             joint_feats_output = self.classifiers["joint_feats"](embeddings, joint_feats)
-            result["joint_feats"] = joint_feats_output['preds']
-            result["loss"] += joint_feats_output['loss']
+            output["joint_feats"] = joint_feats_output['preds']
+            output["loss"] += joint_feats_output['loss']
 
         # Predict syntax.
         if "syntax" in self.classifiers:
-            padding_mask = build_padding_mask(result["words"], self.device)
-            null_mask = build_null_mask(result["words"], self.device)
+            padding_mask = build_padding_mask(output["words"], self.device)
+            null_mask = build_null_mask(output["words"], self.device)
             deps_output = self.classifiers["syntax"](
                 embeddings,
                 deps_ud,
                 deps_eud,
-                mask_ud=(padding_mask & ~null_mask),
-                mask_eud=padding_mask
+                null_mask,
+                padding_mask
             )
-            result["deps_ud"] = deps_output['preds_ud']
-            result["deps_eud"] = deps_output['preds_eud']
-            result["loss"] += deps_output['loss_ud'] + deps_output['loss_eud']
+            output["deps_ud"] = deps_output['preds_ud']
+            output["deps_eud"] = deps_output['preds_eud']
+            output["loss"] += deps_output['loss_ud'] + deps_output['loss_eud']
 
         # Predict miscellaneous features.
         if "misc" in self.classifiers:
             misc_output = self.classifiers["misc"](embeddings, miscs)
-            result["miscs"] = misc_output['preds']
-            result["loss"] += misc_output['loss']
+            output["miscs"] = misc_output['preds']
+            output["loss"] += misc_output['loss']
 
         # Predict semantics.
         if "deepslot" in self.classifiers:
             deepslot_output = self.classifiers["deepslot"](embeddings, deepslots)
-            result["deepslots"] = deepslot_output['preds']
-            result["loss"] += deepslot_output['loss']
+            output["deepslots"] = deepslot_output['preds']
+            output["loss"] += deepslot_output['loss']
 
         if "semclass" in self.classifiers:
             semclass_output = self.classifiers["semclass"](embeddings, semclasses)
-            result["semclasses"] = semclass_output['preds']
-            result["loss"] += semclass_output['loss']
+            output["semclasses"] = semclass_output['preds']
+            output["loss"] += semclass_output['loss']
 
-        return CobaldParserOutput(**result)
+        return output

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:09fd75fcff1724f060e15c6d1fd2cd167eab8304208d648ba480d453b4974510
-size 5496
+oid sha256:2d8dec73b5638e57d0bca9bd4ee05cd11ce5aba98bc59f80b4e16231e6e7403f
+size 5905

utils.py

@@ -21,7 +21,7 @@ def build_padding_mask(sentences: list[list[str]], device) -> Tensor:
     return _build_condition_mask(sentences, condition_fn=lambda word: True, device=device)
 
 def build_null_mask(sentences: list[list[str]], device) -> Tensor:
-    return _build_condition_mask(sentences, condition_fn=lambda word: word == "#NULL", device=device)
+    return _build_condition_mask(sentences, condition_fn=lambda word: word != "#NULL", device=device)
 
 
 def pairwise_mask(masks1d: Tensor) -> Tensor: