Update app.py

app.py

@@ -71,6 +71,16 @@ from tqdm.autonotebook import tqdm
 import numpy as np
 import re
 
+import yake
+
+language = "en"
+max_ngram_size = 3
+deduplication_threshold = 0.9
+deduplication_algo = 'seqm'
+windowSize = 3
+numOfKeywords = 3
+
+custom_kw_extractor = yake.KeywordExtractor(lan=language, n=max_ngram_size, dedupLim=deduplication_threshold, dedupFunc=deduplication_algo, windowsSize=windowSize, top=numOfKeywords, features=None)
 
 # We lower case our text and remove stop-words from indexing
 def bm25_tokenizer(text):
@@ -88,6 +98,9 @@ for passage in tqdm(passages):
 
 bm25 = BM25Okapi(tokenized_corpus)
 
+def word_len(s):
+    return len([i for i in s.split(' ') if i])
+
 # This function will search all wikipedia articles for passages that
 # answer the query
 def search(query):
@@ -163,7 +176,24 @@ def search(query):
         #st.write(rs_final.strip())
         res.append(rs_final.strip())
 
-    st.write(res[0:maxtags_sidebar])
+    #st.write(res[0:maxtags_sidebar])
+
+    res_clean = []
+    for out in result:
+      if len(out) > 20:
+        keywords = custom_kw_extractor.extract_keywords(out)
+        for key in keywords:
+          res_clean.append(key[0])
+      else:
+        res_clean.append(out)
+
+    show_out = []
+    for i in res_clean:
+      num = word_len(i)
+      if num > 1:
+        show_out.append(i)
+
+    st.write(show_out[0:maxtags_sidebar])
 
 st.write("## Results:")
 if st.button('Generated Expansion'):