remove previous offsets

generic_ner.py

@@ -132,8 +132,8 @@ def get_entities(tokens, tags, confidences, text):
                         "surface": text[
                             entity_start_position:entity_end_position
                         ],  # original_string,
-                        "start": entity_start_position,
-                        "end": entity_end_position,
+                        "lOffset": entity_start_position,
+                        "rOffset": entity_end_position,
                     }
                 )
 
@@ -224,7 +224,7 @@ def attach_comp_to_closest(entities):
         # Find the closest non-"comp.function" entity that is valid for attaching
         for other_entity in other_entities:
             distance = abs(
-                comp_entity["start"] - other_entity["end"]
+                comp_entity["lOffset"] - other_entity["rOffset"]
             )  # Calculate the distance
 
             # Ensure that the other entity's type is valid for the attachment
@@ -363,12 +363,6 @@ def remove_included_entities(entities):
     return final_entities
 
 
-# from stopwordsiso import stopwords
-#
-# stop_words = stopwords(["en", "fr", "de"])
-# get the stopwords from nltk
-
-
 def remove_trailing_stopwords(entities):
     """
     This function removes stopwords and punctuation from both the beginning and end of each entity's text
@@ -444,7 +438,10 @@ class MultitaskTokenClassificationPipeline(Pipeline):
 
     def is_within(self, entity1, entity2):
         """Check if entity1 is fully within the bounds of entity2."""
-        return entity1["start"] >= entity2["start"] and entity1["end"] <= entity2["end"]
+        return (
+            entity1["lOffset"] >= entity2["lOffset"]
+            and entity1["rOffset"] <= entity2["rOffset"]
+        )
 
     def postprocess(self, outputs, **kwargs):
         """
@@ -486,12 +483,7 @@ class MultitaskTokenClassificationPipeline(Pipeline):
 
         # print("After remove_included_entities:")
         all_entities = remove_included_entities(all_entities)
-
         all_entities = remove_trailing_stopwords(all_entities)
-        # print("After remove_trailing_stopwords:")
-        # pprint(all_entities)
-        # Attach "comp.function" entities to the closest non-"comp.function" entity
-        # print("After postprocess_entities")
         all_entities = postprocess_entities(all_entities)
 
         # print("After attach_comp_to_closest:")