Update app.py

app.py

@@ -47,10 +47,12 @@ class RAGEvaluator:
         self.current_dataset = None
         self.test_samples = []
         
-    def load_dataset(self, dataset_name: str, num_samples: int = 50):
+    def load_dataset(self, dataset_name: str, num_samples: int = 5): 
+        """Load a smaller subset of questions"""
         if dataset_name == "squad":
             dataset = load_dataset("squad_v2", split="validation")
-            samples = dataset.select(range(num_samples))
+            # Select diverse questions based on length and type
+            samples = dataset.select(range(0, 1000, 100))[:num_samples]  # Take 10 spaced-out samples
             self.test_samples = [
                 {
                     "question": sample["question"],
@@ -62,7 +64,7 @@ class RAGEvaluator:
             ]
         elif dataset_name == "msmarco":
             dataset = load_dataset("ms_marco", "v2.1", split="train")
-            samples = dataset.select(range(num_samples))
+            samples = dataset.select(range(0, 1000, 100))[:num_samples]
             self.test_samples = [
                 {
                     "question": sample["query"],
@@ -76,40 +78,60 @@ class RAGEvaluator:
         return self.test_samples
 
     def evaluate_configuration(self, vector_db, qa_chain, splitting_strategy: str, chunk_size: str) -> Dict:
+        """Evaluate with progress tracking"""
         if not self.test_samples:
             return {"error": "No dataset loaded"}
             
         results = []
-        for sample in self.test_samples:
-            response = qa_chain.invoke({
-                "question": sample["question"],
-                "chat_history": []
-            })
+        total_questions = len(self.test_samples)
+        
+        # Add progress tracking
+        for i, sample in enumerate(self.test_samples):
+            print(f"Evaluating question {i+1}/{total_questions}")
             
-            results.append({
-                "question": sample["question"],
-                "answer": response["answer"],
-                "contexts": [doc.page_content for doc in response["source_documents"]],
-                "ground_truths": [sample["ground_truth"]]
-            })
+            try:
+                response = qa_chain.invoke({
+                    "question": sample["question"],
+                    "chat_history": []
+                })
+                
+                results.append({
+                    "question": sample["question"],
+                    "answer": response["answer"],
+                    "contexts": [doc.page_content for doc in response["source_documents"]],
+                    "ground_truths": [sample["ground_truth"]]
+                })
+            except Exception as e:
+                print(f"Error processing question {i+1}: {str(e)}")
+                continue
         
+        # Calculate RAGAS metrics
         eval_dataset = Dataset.from_list(results)
         metrics = [ContextRecall(), AnswerRelevancy(), Faithfulness(), ContextPrecision()]
-        scores = evaluate(eval_dataset, metrics=metrics)
         
-        return {
-            "configuration": f"{splitting_strategy}_{chunk_size}",
-            "context_recall": float(scores['context_recall']),
-            "answer_relevancy": float(scores['answer_relevancy']),
-            "faithfulness": float(scores['faithfulness']),
-            "context_precision": float(scores['context_precision']),
-            "average_score": float(np.mean([
-                scores['context_recall'],
-                scores['answer_relevancy'],
-                scores['faithfulness'],
-                scores['context_precision']
-            ]))
-        }
+        try:
+            scores = evaluate(eval_dataset, metrics=metrics)
+            
+            return {
+                "configuration": f"{splitting_strategy}_{chunk_size}",
+                "questions_evaluated": len(results),
+                "context_recall": float(scores['context_recall']),
+                "answer_relevancy": float(scores['answer_relevancy']),
+                "faithfulness": float(scores['faithfulness']),
+                "context_precision": float(scores['context_precision']),
+                "average_score": float(np.mean([
+                    scores['context_recall'],
+                    scores['answer_relevancy'],
+                    scores['faithfulness'],
+                    scores['context_precision']
+                ]))
+            }
+        except Exception as e:
+            return {
+                "configuration": f"{splitting_strategy}_{chunk_size}",
+                "error": str(e),
+                "questions_evaluated": len(results)
+            }
 
 # Text splitting and database functions
 def get_text_splitter(strategy: str, chunk_size: int = 1024, chunk_overlap: int = 64):