Duplicate from HarryLee/QueryExpansion

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+000000000001.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,13 @@
+---
+title: QueryExpansion
+emoji: 👁
+colorFrom: pink
+colorTo: indigo
+sdk: streamlit
+sdk_version: 1.17.0
+app_file: app.py
+pinned: false
+duplicated_from: HarryLee/QueryExpansion
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,100 @@
+import streamlit as st
+from streamlit_tags import st_tags, st_tags_sidebar
+from keytotext import pipeline
+from PIL import Image
+
+import json
+from sentence_transformers import SentenceTransformer, CrossEncoder, util
+import gzip
+import os
+import torch
+import pickle
+
+############
+## Main page
+############
+
+st.write("# Code for Query Expansion")
+
+st.markdown("***Idea is to build a model which will take query as inputs and generate expansion information as outputs.***")
+image = Image.open('top.png')
+st.image(image)
+
+st.sidebar.write("# Parameter Selection")
+maxtags_sidebar = st.sidebar.slider('Number of query allowed?', 1, 10, 1, key='ehikwegrjifbwreuk')
+user_query = st_tags(
+    label='# Enter Query:',
+    text='Press enter to add more',
+    value=['Mother'],
+    suggestions=['five', 'six', 'seven', 'eight', 'nine', 'three', 'eleven', 'ten', 'four'],
+    maxtags=maxtags_sidebar,
+    key="aljnf")
+
+# Add selectbox in streamlit
+option1 = st.sidebar.selectbox(
+     'Which transformers model would you like to be selected?',
+     ('multi-qa-MiniLM-L6-cos-v1','null','null'))
+
+option2 = st.sidebar.selectbox(
+     'Which corss-encoder model would you like to be selected?',
+     ('cross-encoder/ms-marco-MiniLM-L-6-v2','null','null'))
+
+st.sidebar.success("Load Successfully!")
+
+#if not torch.cuda.is_available():
+#    print("Warning: No GPU found. Please add GPU to your notebook")
+
+#We use the Bi-Encoder to encode all passages, so that we can use it with sematic search
+bi_encoder = SentenceTransformer(option1)
+bi_encoder.max_seq_length = 256    #Truncate long passages to 256 tokens
+top_k = 32                          #Number of passages we want to retrieve with the bi-encoder
+
+#The bi-encoder will retrieve 100 documents. We use a cross-encoder, to re-rank the results list to improve the quality
+cross_encoder = CrossEncoder(option2)
+
+# load pre-train embeedings files
+embedding_cache_path = 'etsy-embeddings-cpu.pkl'
+print("Load pre-computed embeddings from disc")
+with open(embedding_cache_path, "rb") as fIn:
+  cache_data = pickle.load(fIn)
+  #corpus_sentences = cache_data['sentences']
+  corpus_embeddings = cache_data['embeddings']
+
+# This function will search all wikipedia articles for passages that
+# answer the query
+def search(query):
+    print("Input question:", query)
+    ##### Sematic Search #####
+    # Encode the query using the bi-encoder and find potentially relevant passages
+    query_embedding = bi_encoder.encode(query, convert_to_tensor=True)
+    #query_embedding = query_embedding.cuda()
+    hits = util.semantic_search(query_embedding, corpus_embeddings, top_k=top_k)
+    hits = hits[0]  # Get the hits for the first query
+
+    ##### Re-Ranking #####
+    # Now, score all retrieved passages with the cross_encoder
+    cross_inp = [[query, passages[hit['corpus_id']]] for hit in hits]
+    cross_scores = cross_encoder.predict(cross_inp)
+
+    # Sort results by the cross-encoder scores
+    for idx in range(len(cross_scores)):
+        hits[idx]['cross-score'] = cross_scores[idx]
+
+    # Output of top-10 hits from bi-encoder
+    print("\n-------------------------\n")
+    print("Top-10 Bi-Encoder Retrieval hits")
+    hits = sorted(hits, key=lambda x: x['score'], reverse=True)
+    for hit in hits[0:10]:
+        print("\t{:.3f}\t{}".format(hit['score'], passages[hit['corpus_id']].replace("\n", " ")))
+
+    # Output of top-10 hits from re-ranker
+    print("\n-------------------------\n")
+    print("Top-10 Cross-Encoder Re-ranker hits")
+    hits = sorted(hits, key=lambda x: x['cross-score'], reverse=True)
+    for hit in hits[0:10]:
+        print("\t{:.3f}\t{}".format(hit['cross-score'], passages[hit['corpus_id']].replace("\n", " ")))
+
+st.write("## Results:")
+if st.button('Generate Sentence'):
+    out = search(query = user_query)
+    st.success(out)

etsy-embeddings-cpu.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a8eb36f4ec40a7d1cb382376afc38cac7caed6104bbaf5a8b28f8a98ba18cb5
+size 456491627

requirements.txt

@@ -0,0 +1,7 @@
+streamlit==0.82.0
+streamlit_tags
+pyarrow
+keytotext
+opencv-python-headless
+sentence-transformers
+rank_bm25