Duplicate from kaxap/r-jokes-multilingual-e5-large

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,13 @@
+---
+title: Semantic search on r/jokes using Multilingual E5 Large
+emoji: 🐢
+colorFrom: indigo
+colorTo: green
+sdk: gradio
+sdk_version: 3.39.0
+app_file: app.py
+pinned: false
+duplicated_from: kaxap/r-jokes-multilingual-e5-large
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,59 @@
+import gradio as gr
+
+import pandas as pd
+import numpy as np
+
+import torch.nn.functional as F
+
+from torch import Tensor
+from transformers import AutoTokenizer, AutoModel
+from sklearn.metrics.pairwise import cosine_similarity
+
+
+def average_pool(last_hidden_states: Tensor,
+                 attention_mask: Tensor) -> Tensor:
+    last_hidden = last_hidden_states.masked_fill(~attention_mask[..., None].bool(), 0.0)
+    return last_hidden.sum(dim=1) / attention_mask.sum(dim=1)[..., None]
+
+
+df = pd.read_csv('rjokes.csv')
+data_embeddings = np.load("rjokes-embeddings.npy")
+
+print("loading the model...")
+tokenizer = AutoTokenizer.from_pretrained('intfloat/multilingual-e5-large')
+model = AutoModel.from_pretrained('intfloat/multilingual-e5-large')
+
+with gr.Blocks() as demo:
+    chatbot = gr.Chatbot()
+    msg = gr.Textbox(label="r/jokes semantic search query", placeholder="for example, \"programming and religion\"")
+    clear = gr.ClearButton([msg, chatbot])
+
+    def respond(message, chat_history):
+        batch_dict = tokenizer(["query: " + message], max_length=512, padding=True, truncation=True, return_tensors='pt')
+    
+        outputs = model(**batch_dict)
+        input_embedding = average_pool(outputs.last_hidden_state, batch_dict['attention_mask'])
+    
+        # normalize embeddings
+        input_embedding = F.normalize(input_embedding, p=2, dim=1)
+        input_embedding = input_embedding[0].tolist()
+    
+        # Compute cosine similarities
+        input_embedding = np.array(input_embedding).reshape(1, -1)
+        cos_similarities = cosine_similarity(data_embeddings, input_embedding).flatten()
+    
+        # Get top k similar points' indices
+        k = 5  # replace with your value of k
+        top_k_idx = cos_similarities.argsort()[-k:][::-1]
+    
+        # Get corresponding 'text' for top k similar points
+        top_k_text = df['text'].iloc[top_k_idx].tolist()
+        
+        bot_message = "\n".join(f"{i+1}. {top_k_text[i]}" for i in range(len(top_k_text)))
+    
+        chat_history.append((message, bot_message))
+        return "", chat_history
+
+    msg.submit(respond, [msg, chatbot], [msg, chatbot])
+
+demo.launch()

requirements.txt

@@ -0,0 +1,5 @@
+torch
+numpy
+pandas
+transformers
+scikit-learn

rjokes-embeddings.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20e61c6a0ebb6fecf9329b1f77ba6d01ff570e8e846fca16cab486830a7350bf
+size 354214016