Upload 4 files

app.py

@@ -14,7 +14,7 @@ st.markdown(
     <style>
     .stTextArea textarea {
         background-color: white !important;
-        font-size: 20px !important;
+        font-size: 24px !important;
         color: black !important;
     }
     </style>
@@ -82,19 +82,19 @@ if "query_dataset" not in st.session_state:
     st.session_state.query_dataset = ''
 
 recent_questions = load_recent_questions()
+print(recent_questions)
 
-# for visualization
-
-# response_time = [q["response_time"] for q in recent_data["questions"]]
-# labels = [f"Q{i+1}" for i in range(len(response_time))]  # Labels for X-axis
-
-# fig, ax = plt.subplots()
-# ax.set_xlabel("Recent Questions")
-# ax.set_ylabel("Time Taken for Response")
-# ax.legend()
-# st.sidebar.pyplot(fig)
 if recent_questions and "questions" in recent_questions and recent_questions["questions"]:
     recent_qns = list(reversed(recent_questions["questions"]))
+
+    print(recent_qns)
+
+    # Display Recent Questions
+    st.sidebar.title("Recent Questions")
+    for q in recent_qns:  # Show latest first
+        st.sidebar.write(f"🔹 {q['question']}")
+
+    st.sidebar.markdown("---")
     st.sidebar.title("Analytics")
 
     # Extract response times and labels
@@ -119,18 +119,6 @@ if recent_questions and "questions" in recent_questions and recent_questions["qu
         st.sidebar.write(f"🔹 {q['question']}")
 else:
     st.sidebar.write("No recent questions")
- # Separator
-
-# Streamlit Sidebar for Recent Questions
-
-
-# Submit Button
-# if st.button("Submit"):
-#     start_time = time.time()
-#     st.session_state.retrieved_documents = retrieve_documents_hybrid(question, 10)  
-#     st.session_state.response = generate_response_from_document(question, st.session_state.retrieved_documents)
-#     end_time = time.time()
-#     st.session_state.time_taken_for_response = end_time - start_time
 
 if st.button("Submit"):
     start_time = time.time()
@@ -140,7 +128,12 @@ if st.button("Submit"):
     st.session_state.response = generate_response_from_document(question, st.session_state.retrieved_documents)
     end_time = time.time()
     st.session_state.time_taken_for_response = end_time - start_time
-    save_recent_question(question, st.session_state.time_taken_for_response)
+
+    # Store in session state
+    st.session_state.recent_questions.append({
+        "question": question,
+        "response_time": st.session_state.time_taken_for_response
+    })
 
 # Display stored response
 st.subheader("Response")
@@ -164,10 +157,15 @@ col1, col2 = st.columns([1, 3])  # Creating two columns for button and metrics d
 with col1:
     if st.button("Show Metrics"):     
         st.session_state.metrics = calculate_metrics(question, st.session_state.query_dataset, st.session_state.response, st.session_state.retrieved_documents, st.session_state.time_taken_for_response)
+        metrics_ = st.session_state.metrics
     else:
         metrics_ = {}
 
 with col2:
     #st.text_area("Metrics:", value=metrics, height=100, disabled=True)
-    st.json(st.session_state.metrics)
+    if len(metrics_) > 0: 
+        st.json(metrics_)
+
+save_recent_question(question, st.session_state.metrics)
+
 

data_processing.py

@@ -21,7 +21,6 @@ embedding_model = HuggingFaceEmbeddings(
 reranker = CrossEncoder("cross-encoder/ms-marco-MiniLM-L-6-v2")
 query_dataset_data = {}
 
-# File path for storing recently asked questions and metrics
 RECENT_QUESTIONS_FILE = "data_local/recent_questions.json"
 
 # Ensure the file exists and initialize if empty
@@ -36,10 +35,7 @@ chunk_docs = []
 documents = [] 
 query_dataset_data = {}
 
-# Ensure data directory exists
-os.makedirs("data_local", exist_ok=True)
-
-# Initialize a text splitter
+# Text splitter
 text_splitter = RecursiveCharacterTextSplitter(
     chunk_size=1024,
     chunk_overlap=100
@@ -55,14 +51,12 @@ def create_faiss_index(dataset):
 
     for split in ragbench_dataset.keys():
         for row in ragbench_dataset[split]:
-            # Ensure document is a string before appending
             doc = row["documents"]
             if isinstance(doc, list):
-                # If doc is a list, join its elements into a single string
                 doc = " ".join(doc)
-            documents.append(doc)  # Extract document text
-            # Chunking
-
+            documents.append(doc)  #
+            
+    # Chunking
     chunked_documents = chunk_documents(documents)
 
     # Save documents in JSON (metadata storage)
@@ -76,7 +70,6 @@ def create_faiss_index(dataset):
     # Convert embeddings to a NumPy array
     embeddings_np = np.array(embeddings, dtype=np.float32)
 
-
     # Save FAISS index
     index = faiss.IndexHNSWFlat(embeddings_np.shape[1], 32)  # 32 is the graph size
     index.add(embeddings_np)
@@ -141,17 +134,16 @@ def load_recent_questions():
     if os.path.exists(RECENT_QUESTIONS_FILE):
         with open(RECENT_QUESTIONS_FILE, "r") as file:
             return json.load(file)
-    return {"questions": []}  # Default structure if file doesn't exist
+    return {"questions": []}  
 
-def save_recent_question(question, response_time):
+def save_recent_question(question, metrics):
     data = load_recent_questions()
 
-    #data["questions"] = [q for q in data["questions"] if q["question"] != question]
     if "question" in data["questions"] and question not in data["questions"]["question"]:
         # Append new question & metrics
         data["questions"].append({
             "question": question,
-            "response_time": response_time
+            "metrics": metrics
         })
 
     # Keep only the last 5 questions

evaluation.py

@@ -101,13 +101,13 @@ def calculate_metrics(question, q_dataset, response, docs, time_taken):
 
     # Predicted metrics
     predicted_metrics = {
-        "RAG_model_response": response,
-        "ground_truth": ground_truth_answer,
         "context_relevance": context_relevance(question, docs),
         "context_utilization": context_utilization(response, docs),
         "completeness": completeness(response, ground_truth_answer),
         "adherence": adherence(response, docs),
-        "response_time": time_taken
+        "response_time": time_taken,
+        "ground_truth": ground_truth_answer,
+        "RAG_model_response": response
     }
     return predicted_metrics
 
@@ -115,7 +115,8 @@ def retrieve_ground_truths(question, dataset):
     for split_name, instances in dataset.items():
         print(f"Processing {split_name} split")
         for instance in instances:
-            if instance['question'] == question:
+            #if instance['question'] == question:
+            if is_similar(instance['question'], question):
                 instance_id = instance['id']
                 instance_response = instance['response']
                 # ground_truth_metrics = {
@@ -128,4 +129,10 @@ def retrieve_ground_truths(question, dataset):
                 print(f"ID: {instance_id}, Response: {instance_response}")
                 return instance_response  # Return ground truth response immediately
 
-    return None  # Return None if no match is found
+    return None  
+
+def is_similar(question1, question2, threshold=0.85):
+    vectorizer = TfidfVectorizer()
+    vectors = vectorizer.fit_transform([question1, question2])
+    similarity = cosine_similarity(vectors[0], vectors[1])[0][0]
+    return similarity >= threshold 

retrieval.py

@@ -5,11 +5,17 @@ import faiss
 from rank_bm25 import BM25Okapi
 from data_processing import embedding_model
 from sentence_transformers import CrossEncoder
+from nltk.tokenize import word_tokenize
+import string
 
 reranker = CrossEncoder("cross-encoder/ms-marco-MiniLM-L-6-v2")
 
 retrieved_docs = None
 
+# Tokenize the documents and remove punctuation
+def preprocess(doc):
+    return [word.lower() for word in word_tokenize(doc) if word not in string.punctuation]
+
 def retrieve_documents_hybrid(query, q_dataset, top_k=5):
     with open( f"data_local/{q_dataset}_chunked_docs.json", "r") as f:
         chunked_documents = json.load(f)  # Contains all documents for this dataset
@@ -18,29 +24,48 @@ def retrieve_documents_hybrid(query, q_dataset, top_k=5):
     index = faiss.read_index(faiss_index_path)
 
     # Tokenize documents for BM25
-    tokenized_docs = [doc.split() for doc in chunked_documents]
+    tokenized_docs = [preprocess(doc) for doc in chunked_documents]
     bm25 = BM25Okapi(tokenized_docs)
 
     query_embedding = np.array(embedding_model.embed_documents([query]), dtype=np.float32)
     query_embedding = query_embedding.reshape(1, -1)
 
     # FAISS Search
-    _, nearest_indices = index.search(query_embedding, top_k)
-    faiss_docs = [chunked_documents[i] for i in nearest_indices[0]]
+    faiss_distances, faiss_indices = index.search(query_embedding, top_k)
+    faiss_docs = [chunked_documents[i] for i in faiss_indices[0]]
 
     # BM25 Search  
-    tokenized_query = query.split()
+    tokenized_query = preprocess(query)
     bm25_scores = bm25.get_scores(tokenized_query)
     bm25_top_indices = np.argsort(bm25_scores)[::-1][:top_k]
     bm25_docs = [chunked_documents[i] for i in bm25_top_indices]
 
-    # Merge FAISS + BM25 Results
-    retrieved_docs = list(set(faiss_docs + bm25_docs))[:top_k]
-    
-    reranked_docs = rerank_documents(query, retrieved_docs)
+    # Combine FAISS + BM25 scores and retrieve docs
+    combined_results = set(bm25_top_indices).union(set(faiss_indices[0])) 
+        
+    combined_scores = rerank_docs_bm25faiss_scores(combined_results,bm25_scores, faiss_distances,faiss_indices)
+    reranked_docs = [chunked_documents[result[0]] for result in combined_scores[:top_k]]
+
+    # Merge FAISS + BM25 Results and re-rank
+    #retrieved_docs = list(set(faiss_docs + bm25_docs))[:top_k]
+    #reranked_docs = rerank_documents(query, retrieved_docs)
 
     return reranked_docs
 
+def rerank_docs_bm25faiss_scores(combined_results_,bm25_scores_, faiss_distances_,faiss_indices_):
+    final_results = []
+    for idx in combined_results_:
+        # Combine BM25 score and FAISS score for ranking (this could be more sophisticated)
+        bm25_score = bm25_scores_[idx]
+        faiss_score = 1 / (1 + faiss_distances_[0][np.where(faiss_indices_[0] == idx)])  # Inverse distance for relevance
+        final_results.append((idx, bm25_score, faiss_score))
+
+    # Sort final results by combined score (you can adjust the ranking strategy here)
+    final_results.sort(key=lambda x: (x[1] + x[2]), reverse=True)
+
+    return final_results
+    
+
 # Retrieval Function
 # def retrieve_documents(query, top_k=5):
 #     query_dataset = find_query_dataset(query)
@@ -62,9 +87,8 @@ def retrieve_documents_hybrid(query, q_dataset, top_k=5):
 
 def remove_duplicate_documents(documents):
     unique_documents = []
-    seen_documents = set()  # To keep track of seen documents
+    seen_documents = set() 
     for doc in documents:
-        # Using the page_content as a unique identifier for deduplication
         doc_content = doc.page_content
         if doc_content not in seen_documents:
             unique_documents.append(doc)