Update app.py

app.py

@@ -90,6 +90,7 @@ from allennlp.predictors.predictor import Predictor
 import allennlp_models.tagging
 predictor = Predictor.from_path("structured-prediction-srl-bert.tar.gz")
 
+#---------------------------------------------------------------
 #---------------------------------------------------------------
 def claim(text):
     import re
@@ -148,163 +149,150 @@ def claim(text):
     #----------FOR COLUMN "WHO"------------#
     df['who'] = ''
     for j in range(len(df['modified'])):
-      val_list = []
-      val_string = ''
-      for k,v in df['modified'][j].items():
-        # print(type(v))
-        val_list.append(v)
-
-      who = []
-      for indx in range(len(val_list)):
-        val_string = val_list[indx]
-        pos = val_string.find("who: ")
-        substr = ''
-
-        if pos != -1:
-          for i in range(pos+5, len(val_string)):
-            if val_string[i] == "]":
-              break
+        val_list = []
+        val_string = ''
+        for k,v in df['modified'][j].items():
+            val_list.append(v)
+
+        who = set() # use set to remove duplicates
+        for indx in range(len(val_list)):
+            val_string = val_list[indx]
+            pos = val_string.find("who: ")
+            substr = ''
+
+            if pos != -1:
+                for i in range(pos+5, len(val_string)):
+                    if val_string[i] == "]":
+                        break
+                    else:
+                        substr = substr + val_string[i]
+                substr = substr.strip() # remove leading/trailing white space
+                pronouns = ['he', 'she', 'they', 'it', 'him', 'her', 'them', 'its', 'himself', 'herself', 'themselves']
+                if substr.lower() not in pronouns and not substr.lower().endswith("'s"): # remove pronouns and possessive pronouns
+                    who.add(substr)
             else:
-              substr = substr + val_string[i]
-        else:
-          pass
-        if len(substr)!= 0: 
-            who.append(substr)
-        else:
-            pass
-    # who=list(set(who))
-    df['who'][j] = "<sep>".join(who)
+                pass
+
+        df['who'][j] = "<sep>".join(who)
+
 #     else:
 #         continue
     #----------FOR COLUMN "WHAT"------------#
     df['what'] = ''
     for j in range(len(df['modified'])):
-      val_list = []
-      val_string = ''
-      for k,v in df['modified'][j].items():
-        # print(type(v))
-        val_list.append(v)
-
-      what = []
-      for indx in range(len(val_list)):
-        val_string = val_list[indx]
-        pos = val_string.find("what: ")
-        substr = ''
-
-        if pos != -1:
-          for i in range(pos+6, len(val_string)):
-            if val_string[i] == "]":
-              break
+        val_list = []
+        val_string = ''
+        for k,v in df['modified'][j].items():
+            val_list.append(v)
+
+        what = set() # use set to remove duplicates
+        for indx in range(len(val_list)):
+            val_string = val_list[indx]
+            pos = val_string.find("what: ")
+            substr = ''
+
+            if pos != -1:
+                for i in range(pos+5, len(val_string)):
+                    if val_string[i] == "]":
+                        break
+                    else:
+                        substr = substr + val_string[i]
+                substr = substr.strip() # remove leading/trailing white space
+                pronouns = ['he', 'she', 'they', 'it', 'him', 'her', 'them', 'its', 'himself', 'herself', 'themselves']
+                if substr.lower() not in pronouns and not substr.lower().endswith("'s"): # remove pronouns and possessive pronouns
+                    what.add(substr)
             else:
-              substr = substr + val_string[i]
-        else:
-          pass
-        if len(substr)!= 0: 
-            what.append(substr)
-        else:
-            pass
-  
-    df['what'][j] = "<sep>".join(what)
-#     else:
-#         continue
+                pass
+
+        df['what'][j] = "<sep>".join(what)
 
     #----------FOR COLUMN "WHY"------------#
     df['why'] = ''
     for j in range(len(df['modified'])):
-      val_list = []
-      val_string = ''
-      for k,v in df['modified'][j].items():
-        # print(type(v))
-        val_list.append(v)
-
-      why = []
-      for indx in range(len(val_list)):
-        val_string = val_list[indx]
-        pos = val_string.find("why: ")
-        substr = ''
-
-        if pos != -1:
-          for i in range(pos+5, len(val_string)):
-            if val_string[i] == "]":
-              break
+        val_list = []
+        val_string = ''
+        for k,v in df['modified'][j].items():
+            val_list.append(v)
+
+        why = set() # use set to remove duplicates
+        for indx in range(len(val_list)):
+            val_string = val_list[indx]
+            pos = val_string.find("why: ")
+            substr = ''
+
+            if pos != -1:
+                for i in range(pos+5, len(val_string)):
+                    if val_string[i] == "]":
+                        break
+                    else:
+                        substr = substr + val_string[i]
+                substr = substr.strip() # remove leading/trailing white space
+                pronouns = ['he', 'she', 'they', 'it', 'him', 'her', 'them', 'its', 'himself', 'herself', 'themselves']
+                if substr.lower() not in pronouns and not substr.lower().endswith("'s"): # remove pronouns and possessive pronouns
+                    why.add(substr)
             else:
-              substr = substr + val_string[i]
-        else:
-          pass
-        if len(substr)!= 0: 
-            why.append(substr)
-        else:
-            pass
-    # why=list(set(why))
+                pass
 
-    df['why'][j] = "<sep>".join(why)
-#     else:
-#         continue
+        df['why'][j] = "<sep>".join(why)
 
      #----------FOR COLUMN "WHEN"------------#
     df['when'] = ''
     for j in range(len(df['modified'])):
-      val_list = []
-      val_string = ''
-      for k,v in df['modified'][j].items():
-        # print(type(v))
-        val_list.append(v)
-
-      when = []
-      for indx in range(len(val_list)):
-        val_string = val_list[indx]
-        pos = val_string.find("when: ")
-        substr = ''
-
-        if pos != -1:
-          for i in range(pos+6, len(val_string)):
-            if val_string[i] == "]":
-              break
+        val_list = []
+        val_string = ''
+        for k,v in df['modified'][j].items():
+            val_list.append(v)
+
+        when = set() # use set to remove duplicates
+        for indx in range(len(val_list)):
+            val_string = val_list[indx]
+            pos = val_string.find("when: ")
+            substr = ''
+
+            if pos != -1:
+                for i in range(pos+5, len(val_string)):
+                    if val_string[i] == "]":
+                        break
+                    else:
+                        substr = substr + val_string[i]
+                substr = substr.strip() # remove leading/trailing white space
+                pronouns = ['he', 'she', 'they', 'it', 'him', 'her', 'them', 'its', 'himself', 'herself', 'themselves']
+                if substr.lower() not in pronouns and not substr.lower().endswith("'s"): # remove pronouns and possessive pronouns
+                    when.add(substr)
             else:
-              substr = substr + val_string[i]
-        else:
-          pass
-        if len(substr)!= 0: 
-            when.append(substr)
-        else:
-            pass
-    # when=list(set(when))
+                pass
 
-    df['when'][j] = "<sep>".join(when)
-#     else:
-#         continue
+        df['when'][j] = "<sep>".join(when)
 
 
     #----------FOR COLUMN "WHERE"------------#
     df['where'] = ''
     for j in range(len(df['modified'])):
-      val_list = []
-      val_string = ''
-      for k,v in df['modified'][j].items():
-        # print(type(v))
-        val_list.append(v)
-
-      where = []
-      for indx in range(len(val_list)):
-        val_string = val_list[indx]
-        pos = val_string.find("where: ")
-        substr = ''
-
-        if pos != -1:
-          for i in range(pos+7, len(val_string)):
-            if val_string[i] == "]":
-              break
+        val_list = []
+        val_string = ''
+        for k,v in df['modified'][j].items():
+            val_list.append(v)
+
+        where = set() # use set to remove duplicates
+        for indx in range(len(val_list)):
+            val_string = val_list[indx]
+            pos = val_string.find("where: ")
+            substr = ''
+
+            if pos != -1:
+                for i in range(pos+5, len(val_string)):
+                    if val_string[i] == "]":
+                        break
+                    else:
+                        substr = substr + val_string[i]
+                substr = substr.strip() # remove leading/trailing white space
+                pronouns = ['he', 'she', 'they', 'it', 'him', 'her', 'them', 'its', 'himself', 'herself', 'themselves']
+                if substr.lower() not in pronouns and not substr.lower().endswith("'s"): # remove pronouns and possessive pronouns
+                    where.add(substr)
             else:
-              substr = substr + val_string[i]
-        else:
-          pass
-        if len(substr)!= 0: 
-            where.append(substr)
-        else:
-            pass
-    # where=list(set(where))
+                pass
 
-    df['where'][j] = "<sep>".join(where)
+        df['where'][j] = "<sep>".join(where)
 
 
     data=df[["claim","who","what","why","when","where"]].copy()    
@@ -626,7 +614,6 @@ def gen_qa_where(df):
             list_of_evidence_answer_where="No mention of 'where'in any related documents."
             rouge_l_scores="Not verifiable"
     return list_of_ques_where,list_of_ans_where,rouge_l_scores,list_of_evidence_answer_where
-
 #------------------------------------------------------