Update model

data

@@ -1 +1 @@
-Subproject commit 733ac504b6f80dd11244534aa2820333cd0e0176
+Subproject commit 83ccdae5afe7eaf7f88b0ceb4933544e445b7841

segmenter.ckpt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4bc9ddcfd7a654b4a86d7bb711a9d3e9c126269186033f5bc63695f88d4aaa77
+oid sha256:e616108c71de535ed24386f1aaf2e38ab9cdf0dd123517aa5cef72c9ec019ed9
 size 2665888

utils.py

@@ -76,22 +76,67 @@ def tag_training_data(filename: str):
     return reconstructed_tags
 
 def get_upenn_tags_dict():
-    tagger = PerceptronTagger()
-
-    tags = list(tagger.tagdict.values())
-
-    # https://www.ling.upenn.edu/courses/Fall_2003/ling001/penn_treebank_pos.html
-    tags.extend(["CC", "CD", "DT", "EX", "FW", "IN", "JJ", "JJR", "JJS", "LS", "MD", "NN", "NNS", "NNP", "NNPS", "PDT", "POS", "PRP", "PRP$", "RB", "RBR", "RBS", "RP", "SYM", "TO", "UH", "VB", "VBD", "VBG", "VBN", "VBP", "VBZ", "WDT", "WP", "WP$", "WRB"])
-    tags = list(set(tags))
-    tags.sort()
-    tags.append("BREAK")
-
-    tags_dict = dict()
-
-    for index, tag in enumerate(tags):
-        tags_dict[tag] = index
-
-    return tags_dict
+    # tagger = PerceptronTagger()
+
+    # tags = list(tagger.tagdict.values())
+
+    # # https://www.ling.upenn.edu/courses/Fall_2003/ling001/penn_treebank_pos.html
+    # tags.extend(["CC", "CD", "DT", "EX", "FW", "IN", "JJ", "JJR", "JJS", "LS", "MD", "NN", "NNS", "NNP", "NNPS", "PDT", "POS", "PRP", "PRP$", "RB", "RBR", "RBS", "RP", "SYM", "TO", "UH", "VB", "VBD", "VBG", "VBN", "VBP", "VBZ", "WDT", "WP", "WP$", "WRB"])
+    # tags = list(set(tags))
+    # tags.sort()
+    # tags.append("BREAK")
+
+    # tags_dict = dict()
+
+    # for index, tag in enumerate(tags):
+    #     tags_dict[tag] = index
+
+    return {'#': 0,
+            '$': 1,
+            "''": 2,
+            '(': 3,
+            ')': 4,
+            ',': 5,
+            '.': 6,
+            ':': 7,
+            'CC': 8,
+            'CD': 9,
+            'DT': 10,
+            'EX': 11,
+            'FW': 12,
+            'IN': 13,
+            'JJ': 14,
+            'JJR': 15,
+            'JJS': 16,
+            'LS': 17,
+            'MD': 18,
+            'NN': 19,
+            'NNP': 20,
+            'NNPS': 21,
+            'NNS': 22,
+            'PDT': 23,
+            'POS': 24,
+            'PRP': 25,
+            'PRP$': 26,
+            'RB': 27,
+            'RBR': 28,
+            'RBS': 29,
+            'RP': 30,
+            'SYM': 31,
+            'TO': 32,
+            'UH': 33,
+            'VB': 34,
+            'VBD': 35,
+            'VBG': 36,
+            'VBN': 37,
+            'VBP': 38,
+            'VBZ': 39,
+            'WDT': 40,
+            'WP': 41,
+            'WP$': 42,
+            'WRB': 43,
+            '``': 44,
+            'BREAK': 45}
 
 
 def parse_tags(reconstructed_tags):