Duplicate from hamza50/cicero_semantic_search

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,13 @@
+---
+title: Cicero Semantic Search
+emoji: 🐢
+colorFrom: green
+colorTo: gray
+sdk: gradio
+sdk_version: 3.23.0
+app_file: app.py
+pinned: false
+duplicated_from: hamza50/cicero_semantic_search
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,79 @@
+import gradio as gr
+import pandas as pd
+import tiktoken
+import pandas as pd
+import time
+import spacy
+from spacy.lang.en.stop_words import STOP_WORDS
+from string import punctuation
+from collections import Counter
+from heapq import nlargest
+import nltk
+import numpy as np
+from tqdm import tqdm
+from sentence_transformers import SentenceTransformer, util
+from sentence_transformers import SentenceTransformer, CrossEncoder, util
+import gzip
+import os
+import torch
+import re
+
+from openai.embeddings_utils import get_embedding, cosine_similarity
+import os
+
+
+
+df = pd.read_pickle('entire_data.pkl') #to load 123.pkl back to the dataframe df
+model = SentenceTransformer('all-mpnet-base-v2')
+
+def remove_html_tags(text):
+    clean = re.compile('<.*?>')
+    return re.sub(clean, '', text)
+
+df['content'] = df.content.apply(lambda x: remove_html_tags(x))
+df['summary_html'] = df.summary_html.apply(lambda x: remove_html_tags(x))
+
+def search(query):
+    n = 10
+    query_embedding = model.encode(query)
+    df["similarity"] = df.embedding.apply(lambda x: cosine_similarity(x, query_embedding.reshape(768,-1)))
+
+    results = (df.sort_values("similarity", ascending=False).head(n))
+    r_groupby = pd.DataFrame(results.groupby(['title','url','keywords','summary_html']).similarity.max())
+    #results = results[['title','url','keywords','summary_html']].drop_duplicates()  
+    results = r_groupby.reset_index()
+    results = results.sort_values("similarity", ascending=False)
+    resultlist = []
+    for r in results.index:
+        resultlist.append(
+            {
+              "Title":results.title[r],
+                "url":results.url[r],
+             "score": str(results.similarity[r][0]),
+             "summary": results.summary_html[r][:200],
+                "keywords": results.keywords[r]
+            }
+        )
+        
+    return resultlist
+
+def greet(query):
+   
+    bm25 = search(query)
+    return bm25
+
+examples = [
+    ["Climate Change Challenges in Europe"],
+    ["Philosophy in the world of Minimalism"],
+    ["Hate Speech  vs Freedom of Speech"],
+    ["Articles by Noam Chomsky on US Politics"],
+    ["The importance of values and reflection"]
+    ]
+
+demo = gr.Interface(fn=greet, title="cicero-semantic-search", 
+                     inputs=gr.inputs.Textbox(lines=5, label="what would you like to learn about?"),
+                     outputs="json",examples=examples)
+
+demo.launch()
+
+

entire_data.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d719ff7c8e72ee0f56541a05b3eac5241adb7f19c7237ac3d6546af12f6dde22
+size 51891614

requirements.txt

@@ -0,0 +1,20 @@
+pandas
+scipy
+tqdm
+gensim
+plotly
+scikit-learn
+numpy
+wordcloud
+matplotlib
+openai
+langchain
+faiss-cpu
+tiktoken
+sentence_transformers
+scipy
+tqdm
+matplotlib
+spacy
+https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.2.0/en_core_web_sm-3.2.0-py3-none-any.whl 
+rank-bm25