Update generic_ner.py
Browse files- generic_ner.py +2 -2
generic_ner.py
CHANGED
|
@@ -14,7 +14,7 @@ import torch.nn.functional as F
|
|
| 14 |
import re, string
|
| 15 |
|
| 16 |
stop_words = set(nltk.corpus.stopwords.words("english"))
|
| 17 |
-
DEBUG =
|
| 18 |
punctuation = (
|
| 19 |
string.punctuation
|
| 20 |
+ "«»—…“”"
|
|
@@ -543,7 +543,7 @@ def remove_trailing_stopwords(entities):
|
|
| 543 |
rOffset = entity.get("rOffset", original_len)
|
| 544 |
|
| 545 |
# Remove stopwords and punctuation from the beginning
|
| 546 |
-
print('----', entity_text)
|
| 547 |
if len(entity_text.split()) < 1:
|
| 548 |
continue
|
| 549 |
while entity_text and (
|
|
|
|
| 14 |
import re, string
|
| 15 |
|
| 16 |
stop_words = set(nltk.corpus.stopwords.words("english"))
|
| 17 |
+
DEBUG = False
|
| 18 |
punctuation = (
|
| 19 |
string.punctuation
|
| 20 |
+ "«»—…“”"
|
|
|
|
| 543 |
rOffset = entity.get("rOffset", original_len)
|
| 544 |
|
| 545 |
# Remove stopwords and punctuation from the beginning
|
| 546 |
+
# print('----', entity_text)
|
| 547 |
if len(entity_text.split()) < 1:
|
| 548 |
continue
|
| 549 |
while entity_text and (
|