Update app.py

app.py

@@ -200,10 +200,87 @@ def search(query):
 
     return show_out
 
+def search_nolog(query):
+    total_qe = []
+    ##### BM25 search (lexical search) #####
+    bm25_scores = bm25.get_scores(bm25_tokenizer(query))
+    top_n = np.argpartition(bm25_scores, -5)[-5:]
+    bm25_hits = [{'corpus_id': idx, 'score': bm25_scores[idx]} for idx in top_n]
+    bm25_hits = sorted(bm25_hits, key=lambda x: x['score'], reverse=True)
+    
+    qe_string = []
+    for hit in bm25_hits[0:1000]:
+      if passages[hit['corpus_id']].replace("\n", " ") not in qe_string:
+        qe_string.append(passages[hit['corpus_id']].replace("\n", ""))
+
+    sub_string = []
+    for item in qe_string:
+      for sub_item in item.split(","):
+        sub_string.append(sub_item)
+    total_qe.append(sub_string)
+
+    ##### Sematic Search #####
+    # Encode the query using the bi-encoder and find potentially relevant passages
+    query_embedding = bi_encoder.encode(query, convert_to_tensor=True)
+    hits = util.semantic_search(query_embedding, corpus_embeddings, top_k=top_k)
+    hits = hits[0]  # Get the hits for the first query
+
+    ##### Re-Ranking #####
+    # Now, score all retrieved passages with the cross_encoder
+    cross_inp = [[query, passages[hit['corpus_id']]] for hit in hits]
+    cross_scores = cross_encoder.predict(cross_inp)
+
+    # Sort results by the cross-encoder scores
+    for idx in range(len(cross_scores)):
+        hits[idx]['cross-score'] = cross_scores[idx]
+
+    # Output of top-10 hits from bi-encoder
+    hits = sorted(hits, key=lambda x: x['score'], reverse=True)
+    qe_string = []
+    for hit in hits[0:1000]:
+      if passages[hit['corpus_id']].replace("\n", " ") not in qe_string:
+        qe_string.append(passages[hit['corpus_id']].replace("\n", ""))
+    total_qe.append(qe_string)
+
+    # Output of top-10 hits from re-ranker
+    hits = sorted(hits, key=lambda x: x['cross-score'], reverse=True)
+    qe_string = []
+    for hit in hits[0:1000]:
+      if passages[hit['corpus_id']].replace("\n", " ") not in qe_string:
+        qe_string.append(passages[hit['corpus_id']].replace("\n", ""))
+    total_qe.append(qe_string)
+
+    # Total Results
+    total_qe.append(qe_string)
+
+    res = []
+    for sub_list in total_qe:
+      for i in sub_list:
+        rs = re.sub("([^\u0030-\u0039\u0041-\u007a])", ' ', i)
+        rs_final = re.sub("\x20\x20", "\n", rs)
+        res.append(rs_final.strip())
+
+    res_clean = []
+    for out in res:
+      if len(out) > 20:
+        keywords = custom_kw_extractor.extract_keywords(out)
+        for key in keywords:
+          res_clean.append(key[0])
+      else:
+        res_clean.append(out)
+
+    show_out = []
+    for i in res_clean:
+      num = word_len(i)
+      if num > 1:
+        show_out.append(i)
+
+    return show_out
+
 def reranking():
     rerank_list = []
-    rerank_list = search(query = user_query)
-    st.write(rerank_list[0:maxtags_sidebar])
+    rerank_list = search_nolog(query = user_query)
+    random.seed(7)
     random.shuffle(rerank_list[0:maxtags_sidebar])
     for i in rerank_list[0:maxtags_sidebar]:
         st.write(i)