Update generic_ner.py

generic_ner.py

@@ -148,6 +148,18 @@ def realign(
     return words_list, preds_list, confidence_list
 
 
+import re, string
+
+# List of additional "strange" punctuation marks
+additional_punctuation = "‘’“”„«»•–—―‣◦…§¶†‡‰′″〈〉"
+
+
+def add_spaces_around_punctuation(text):
+    # Add a space before and after all punctuation
+    all_punctuation = string.punctuation + additional_punctuation
+    return re.sub(r"([{}])".format(re.escape(all_punctuation)), r" \1 ", text)
+
+
 class MultitaskTokenClassificationPipeline(Pipeline):
 
     def _sanitize_parameters(self, **kwargs):
@@ -166,7 +178,7 @@ class MultitaskTokenClassificationPipeline(Pipeline):
             text, padding="max_length", truncation=True, max_length=512
         )
 
-        text_sentence = tokenize(text)
+        text_sentence = tokenize(add_spaces_around_punctuation(text))
         return tokenized_inputs, text_sentence, text
 
     def _forward(self, inputs):