initial commit

app.py

@@ -0,0 +1,56 @@
+from sentence_transformers import SentenceTransformer, CrossEncoder, util
+import torch
+import pickle
+import pandas as pd
+import gradio as gr
+
+# bi_encoder = SentenceTransformer("microsoft/Multilingual-MiniLM-L12-H384")
+cross_encoder = CrossEncoder("cross-encoder/mmarco-mMiniLMv2-L12-H384-v1")
+# Corpus from quran
+my_file = open("quran-simple-clean.txt", "r",encoding="utf-8")
+data = my_file.read()
+quran = data.split("\n")
+my_file = open("mokhtasar-simple-clean.txt", "r",encoding="utf-8")
+data = my_file.read()
+corpus = data.split("\n")
+del data
+embedder = SentenceTransformer('symanto/sn-xlm-roberta-base-snli-mnli-anli-xnli')
+corpus_embeddings = embedder.encode(corpus, convert_to_tensor=True)
+
+def search(query,top_k=25):
+    print("New query:")
+    print(query)
+    ans=[]
+    ##### Sematic Search #####
+    # Encode the query using the bi-encoder and find potentially relevant passages
+    question_embedding = embedder.encode(query, convert_to_tensor=True)
+    hits = util.semantic_search(question_embedding, corpus_embeddings, top_k=top_k)
+    hits = hits[0]  # Get the hits for the first query
+
+    ##### Re-Ranking #####
+    # Now, score all retrieved passages with the cross_encoder
+    cross_inp = [[query, corpus[hit['corpus_id']]] for hit in hits]
+    cross_scores = cross_encoder.predict(cross_inp)
+
+    # Sort results by the cross-encoder scores
+    for idx in range(len(cross_scores)):
+        hits[idx]['cross-score'] = cross_scores[idx]
+
+    hits = sorted(hits, key=lambda x: x['cross-score'], reverse=True)
+    
+    for idx, hit in enumerate(hits[0:25]):
+        if hit["cross-score"] > 0:
+            ans.append(quran[hit['corpus_id']])
+    if len(ans) == 0:
+        ans.append("لا يوجد نتائج الرجاء تقريب كلمات البحث")
+    return "\n\n".join(ans)
+
+exp=[""]
+
+desc="هذا البحث يعتمد على المختصر في تفسير القرآن في البحث."
+
+inp=gr.inputs.Textbox(lines=1, placeholder=None, default="", label="أدخل كلمات البحث هنا")
+out=gr.outputs.Textbox(type="auto",label="نتائج البحث")
+
+iface = gr.Interface(fn=search, inputs=inp, outputs=out,examples=exp,article=desc,title="البحث بالمعني في القرآن الكريم - باستخدام المختصر في التفسير")
+iface.launch()

requirements.txt

@@ -0,0 +1,4 @@
+sentence_transformers
+torch
+pandas
+gradio