Filtering

app.py

@@ -17,6 +17,7 @@ article = st.text_area('Article to analyze:', value=open("example.txt").read())
 
 seen_entities = []
 seen_surnames = []
+seen_qids = []
 if st.button('Submit'):
     good_ents = []
 
@@ -25,18 +26,24 @@ if st.button('Submit'):
         for ent in doc.ents:
             if ent._.kb_qid is None or ent.label_ not in ["ORG", "PERSON", "GPE"] or ent.text in seen_entities:
                 continue
+            if ent._.nerd_score < 0.5:
+                continue
 
-            if ent.label_ == "PERSON":
-                if len(ent.text.split()) == 1:
-                    # Single name
-                    if ent.text in seen_surnames:
-                        continue
-                else:
-                    # Multipart name
-                    seen_surnames.append(ent.text.split()[-1])
+            if len(ent.text.split()) == 1:
+                # Single name
+                if ent.text in seen_surnames:
+                    continue
+            elif ent.label_ == "PERSON":
+                # Multipart name
+                seen_surnames.append(ent.text.split()[-1])
 
             seen_entities.append(ent.text)
             print((ent.text, ent.label_, ent._.kb_qid, ent._.url_wikidata, ent._.nerd_score))
+
+            if ent._.kb_qid in seen_qids:
+                continue
+            seen_qids.append(ent._.kb_qid)
+
             r = requests.get("https://www.wikidata.org/w/api.php?action=wbgetclaims&format=json&property=P18&entity=" + ent._.kb_qid)
             data = r.json()["claims"]
             if "P18" in data.keys():