added cross encoder

app.py

@@ -7,6 +7,7 @@ from datasets import Features
 from datasets import Value
 from datasets import Dataset
 from sentence_transformers import SentenceTransformer
+from sentence_transformers.cross_encoder import CrossEncoder
 from sklearn.metrics.pairwise import cosine_similarity
 import os
 import gradio as gr
@@ -39,14 +40,20 @@ df = joined_df.copy()
 
 
 model = SentenceTransformer('FDSRashid/QulBERT', token=Secret_token)
+model_CE = CrossEncoder('FDSRashid/QulBERT-CE-2.0', token=Secret_token)
 arr = np.array(df['embed'].to_list())
 
 def find_most_similar_matn(text, n):
-    embed_text = model.encode(araby.strip_diacritics(text))
+    prep_text = araby.strip_diacritics(text)
+    embed_text = model.encode(prep_text)
     cos_sim = cosine_similarity(embed_text.reshape(1, -1), arr)
     indices = np.argsort(cos_sim)[0][-n:]
     matns = df.iloc[indices]
     matns['Similarity'] = cos_sim[0][indices]
+    matns_prep = [araby.strip_diacritics(text) for text in matns['matn']]
+    to_compare = [(i, prep_text) for i in matns_prep]
+    is_taraf = model_CE.predict(to_compare)
+    matns = matns[is_taraf> .5]
     return matns[['Book_Name', 'matn', 'taraf_ID', 'Book_ID', 'Hadith Number', 'Author', 'Similarity']]
 
 with gr.Blocks() as demo: