Upload 2 files

app.py

@@ -78,14 +78,16 @@ if "time_taken_for_response" not in st.session_state:
     st.session_state.time_taken_for_response = "N/A"
 if "metrics" not in st.session_state:
     st.session_state.metrics = {}
-if "query_dataset" not in st.session_state:
+if "query_dataset" not in 
     st.session_state.query_dataset = ''
+if "recent_questions" not in st.session_state:
+    st.session_state.recent_questions = {}
 
-recent_questions = load_recent_questions()
-print(recent_questions)
+st.session_state.recent_questions  = load_recent_questions()
+print(st.session_state.recent_questions )
 
-if recent_questions and "questions" in recent_questions and recent_questions["questions"]:
-    recent_qns = list(reversed(recent_questions["questions"]))
+if st.session_state.recent_questions  and "questions" in st.session_state.recent_questions  and st.session_state.recent_questions ["questions"]:
+    recent_qns = list(reversed(st.session_state.recent_questions ["questions"]))
 
     print(recent_qns)
 
@@ -98,7 +100,7 @@ if recent_questions and "questions" in recent_questions and recent_questions["qu
     st.sidebar.title("Analytics")
 
     # Extract response times and labels
-    response_time = [q["response_time"] for q in recent_qns]
+    response_time = [q['metrics']["response_time"] for q in recent_qns]
     labels = [f"Q{i+1}" for i in range(len(response_time))]
 
     # Plot graph
@@ -130,10 +132,10 @@ if st.button("Submit"):
     st.session_state.time_taken_for_response = end_time - start_time
 
     # Store in session state
-    st.session_state.recent_questions.append({
-        "question": question,
-        "response_time": st.session_state.time_taken_for_response
-    })
+    # st.session_state.recent_questions.append({
+    #     "question": question,
+    #     "response_time": st.session_state.time_taken_for_response
+    # })
 
 # Display stored response
 st.subheader("Response")

retrieval.py

@@ -5,12 +5,11 @@ import faiss
 from rank_bm25 import BM25Okapi
 from data_processing import embedding_model
 from sentence_transformers import CrossEncoder
-import string
-import nltk
 
-import nltk
-nltk.download('punkt')  
-nltk.download('punkt_tab')
+#import string
+# import nltk
+# nltk.download('punkt')  
+# nltk.download('punkt_tab')
 
 from nltk.tokenize import word_tokenize
 
@@ -19,8 +18,8 @@ reranker = CrossEncoder("cross-encoder/ms-marco-MiniLM-L-6-v2")
 retrieved_docs = None
 
 # Tokenize the documents and remove punctuation
-def preprocess(doc):
-    return [word.lower() for word in word_tokenize(doc) if word not in string.punctuation]
+# def preprocess(doc):
+#     return [word.lower() for word in word_tokenize(doc) if word not in string.punctuation]
 
 def retrieve_documents_hybrid(query, q_dataset, top_k=5):
     with open( f"data_local/{q_dataset}_chunked_docs.json", "r") as f:
@@ -30,31 +29,31 @@ def retrieve_documents_hybrid(query, q_dataset, top_k=5):
     index = faiss.read_index(faiss_index_path)
 
     # Tokenize documents for BM25
-    tokenized_docs = [preprocess(doc) for doc in chunked_documents]
+    tokenized_docs = [doc.split() for doc in chunked_documents]
     bm25 = BM25Okapi(tokenized_docs)
 
     query_embedding = np.array(embedding_model.embed_documents([query]), dtype=np.float32)
     query_embedding = query_embedding.reshape(1, -1)
 
     # FAISS Search
-    faiss_distances, faiss_indices = index.search(query_embedding, top_k)
+    _, faiss_indices = index.search(query_embedding, top_k)
     faiss_docs = [chunked_documents[i] for i in faiss_indices[0]]
 
     # BM25 Search  
-    tokenized_query = preprocess(query)
+    tokenized_query = query.split() #preprocess(query)
     bm25_scores = bm25.get_scores(tokenized_query)
     bm25_top_indices = np.argsort(bm25_scores)[::-1][:top_k]
     bm25_docs = [chunked_documents[i] for i in bm25_top_indices]
 
     # Combine FAISS + BM25 scores and retrieve docs
-    combined_results = set(bm25_top_indices).union(set(faiss_indices[0])) 
+    # combined_results = set(bm25_top_indices).union(set(faiss_indices[0])) 
         
-    combined_scores = rerank_docs_bm25faiss_scores(combined_results,bm25_scores, faiss_distances,faiss_indices)
-    reranked_docs = [chunked_documents[result[0]] for result in combined_scores[:top_k]]
+    # combined_scores = rerank_docs_bm25faiss_scores(combined_results,bm25_scores, faiss_distances,faiss_indices)
+    # reranked_docs = [chunked_documents[result[0]] for result in combined_scores[:top_k]]
 
     # Merge FAISS + BM25 Results and re-rank
-    #retrieved_docs = list(set(faiss_docs + bm25_docs))[:top_k]
-    #reranked_docs = rerank_documents(query, retrieved_docs)
+    retrieved_docs = list(set(faiss_docs + bm25_docs))[:top_k]
+    reranked_docs = rerank_documents(query, retrieved_docs)
 
     return reranked_docs