Add encoded corpus

.gitattributes

@@ -32,3 +32,12 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+distiluse-base-multilingual-cased-v1-fiqa-pl-corpus filter=lfs diff=lfs merge=lfs -text
+distiluse-base-multilingual-cased-v1-nfcorpus-pl-corpus filter=lfs diff=lfs merge=lfs -text
+distiluse-base-multilingual-cased-v1-scifact-pl-corpus filter=lfs diff=lfs merge=lfs -text
+mcontriever-nfcorpus-pl-corpus filter=lfs diff=lfs merge=lfs -text
+mcontriever-scifacts-pl-corpus filter=lfs diff=lfs merge=lfs -text
+mcontriever-fiqa-pl-corpus filter=lfs diff=lfs merge=lfs -text
+multilingual-e5-base-nfcorpus-pl-corpus filter=lfs diff=lfs merge=lfs -text
+multilingual-e5-base-scifact-pl-corpus filter=lfs diff=lfs merge=lfs -text
+multilingual-e5-base-fiqa-pl-corpus filter=lfs diff=lfs merge=lfs -text

app.py

@@ -32,20 +32,35 @@ def load_data(dataset_type):
     return queries, corpus
 
 @st.cache_data()
-def bi_encode(bi_enc,passages):
+def bi_encode(bi_encoder_name,passages, dataset_name='scifact-pl'):
     
     global bi_encoder
     #We use the Bi-Encoder to encode all passages, so that we can use it with sematic search
-    bi_encoder = SentenceTransformer(bi_enc,use_auth_token=auth_token)
+    bi_encoder = SentenceTransformer(bi_encoder_name,use_auth_token=auth_token)
     
-    with st.spinner('Encoding passages into a vector space...'):
+    # Thos code would be used if we would embed the passages, but here to make it fast we will load already embedded tensors:
+    # with st.spinner('Encoding passages into a vector space...'):
 
-        if bi_enc == 'intfloat/multilingual-e5-base':
+    #     if bi_encoder_name == 'intfloat/multilingual-e5-base':
             
-            corpus_embeddings = bi_encoder.encode(['passage: ' + sentence for sentence in passages], convert_to_tensor=True)
+    #         corpus_embeddings = bi_encoder.encode(['passage: ' + sentence for sentence in passages], convert_to_tensor=True)
 
-        else:
-            corpus_embeddings = bi_encoder.encode(passages, convert_to_tensor=True)
+    #     else:
+    #         corpus_embeddings = bi_encoder.encode(passages, convert_to_tensor=True)
+
+    with st.spinner('Loading encoded passages...'):
+
+        if bi_encoder_name == "sentence-transformers/distiluse-base-multilingual-cased-v1":
+            name = 'distiluse-base-multilingual-cased-v1'
+
+        elif bi_encoder_name == 'intfloat/multilingual-e5-base':
+            name = 'multilingual-e5-base'
+
+        elif bi_encoder_name == 'nthakur/mcontriever-base-msmarco':
+            name = 'mcontriever'
+
+        corpus_embeddings_name = "-".join([name, dataset_name, "corpus"])
+        corpus_embeddings = torch.load(corpus_embeddings_name)
 
 
     st.success(f"Embeddings computed. Shape: {corpus_embeddings.shape}")
@@ -150,7 +165,10 @@ def search_func(query, bi_encoder_type, top_k=top_k):
 
     ##### Sematic Search #####
     # Encode the query using the bi-encoder and find potentially relevant passages
-    question_embedding = bi_encoder.encode(query, convert_to_tensor=True)
+    if bi_encoder_type == 'intfloat/multilingual-e5-base':
+        question_embedding = bi_encoder.encode("query: " + query, convert_to_tensor=True)
+    else:
+         question_embedding = bi_encoder.encode(query, convert_to_tensor=True)
     question_embedding = question_embedding.cpu()
     hits = util.semantic_search(question_embedding, corpus_embeddings, top_k=top_k,score_function=util.dot_score)
     hits = hits[0]  # Get the hits for the first query

distiluse-base-multilingual-cased-v1-fiqa-pl-corpus

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2312ba328c80b17088d5ae7d704c0362ed086fe58ac7899230047ddc530e8f3f
+size 118043631

distiluse-base-multilingual-cased-v1-nfcorpus-pl-corpus

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:855b2b61bb109bd2c4964af10b924e15a70b0f79e08380dfb9c39697e189c513
+size 7441403

distiluse-base-multilingual-cased-v1-scifact-pl-corpus

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e09e44d2269375c67add8ce85666882551fb547fee01e3259ecdf88842c3301
+size 10615800

mcontriever-fiqa-pl-corpus

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b7e6fe50d931782dd6c24019ebc1db17e5543d247fb4a212c3ef4faf2007b62
+size 177064804

mcontriever-nfcorpus-pl-corpus

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8440623d38b4bd95c56cecbd21d9ca86f479393228a2934f3d0cc3c5edc9cad6
+size 11161456

mcontriever-scifacts-pl-corpus

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:291dd709b7bbe015f1176c9fe37fe8e7a64d55673cac2729a304c7b26a40addd
+size 15923056

multilingual-e5-base-fiqa-pl-corpus

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f8183a87b7c515c54aeddd770e1531cc85fcf99b4f9e3715c0716d020689a8d
+size 177064831

multilingual-e5-base-nfcorpus-pl-corpus

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db3b211fc13de3a311addae59b061049e55a2794639127740c64b0680af8615b
+size 11161547

multilingual-e5-base-scifact-pl-corpus

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4166d06d8037cc70990c3f1ac894aaae0b6187bf69e8f6a6e25d9236edf891ea
+size 15923144