Update app.py

app.py

@@ -53,6 +53,86 @@ def exact_match_score(prediction, ground_truth):
     """Calculate exact match score"""
     return normalize_answer(prediction) == normalize_answer(ground_truth)
 
+def has_answer(answers):
+    """Check if the question has any valid answers"""
+    if not answers or not answers.get("text"):
+        return False
+    
+    answer_texts = answers["text"] if isinstance(answers["text"], list) else [answers["text"]]
+    return any(text.strip() for text in answer_texts)
+
+def get_top_k_predictions(qa_pipeline, question, context, k=3):
+    """Get top-k predictions from the model"""
+    # Get raw model outputs
+    inputs = qa_pipeline.tokenizer(question, context, return_tensors="pt", truncation=True, max_length=512)
+    
+    with torch.no_grad():
+        outputs = qa_pipeline.model(**inputs)
+        start_logits = outputs.start_logits
+        end_logits = outputs.end_logits
+    
+    # Get top-k start and end positions
+    start_scores, start_indices = torch.topk(start_logits.flatten(), k)
+    end_scores, end_indices = torch.topk(end_logits.flatten(), k)
+    
+    predictions = []
+    
+    # Generate all combinations of start and end positions
+    for start_idx in start_indices:
+        for end_idx in end_indices:
+            if start_idx <= end_idx:  # Valid span
+                # Convert to answer text
+                input_ids = inputs["input_ids"][0]
+                answer_tokens = input_ids[start_idx:end_idx + 1]
+                answer_text = qa_pipeline.tokenizer.decode(answer_tokens, skip_special_tokens=True)
+                
+                # Calculate combined score
+                start_score = start_logits[0][start_idx].item()
+                end_score = end_logits[0][end_idx].item()
+                combined_score = start_score + end_score
+                
+                predictions.append({
+                    "answer": answer_text,
+                    "score": combined_score,
+                    "start": start_idx.item(),
+                    "end": end_idx.item()
+                })
+    
+    # Sort by score and return top-k unique answers
+    predictions.sort(key=lambda x: x["score"], reverse=True)
+    unique_answers = []
+    seen_answers = set()
+    
+    for pred in predictions:
+        normalized_answer = normalize_answer(pred["answer"])
+        if normalized_answer not in seen_answers and len(unique_answers) < k:
+            unique_answers.append(pred)
+            seen_answers.add(normalized_answer)
+    
+    return unique_answers
+
+def calculate_top_k_has_ans_f1(predictions, ground_truths, k=1):
+    """Calculate Top-K Has Answer F1 score"""
+    f1_scores = []
+    
+    for preds, gt in zip(predictions, ground_truths):
+        if not has_answer(gt):
+            continue  # Skip questions without answers
+        
+        # Get ground truth text
+        gt_text = gt["text"][0] if isinstance(gt["text"], list) else gt["text"]
+        
+        # Calculate F1 for top-k predictions
+        max_f1 = 0
+        for i in range(min(k, len(preds))):
+            pred_text = preds[i]["answer"]
+            f1 = f1_score_qa(pred_text, gt_text)
+            max_f1 = max(max_f1, f1)
+        
+        f1_scores.append(max_f1)
+    
+    return np.mean(f1_scores) if f1_scores else 0
+
 def evaluate_model():
     # Authenticate with Hugging Face using the token
     hf_token = os.getenv("EVAL_TOKEN")
@@ -88,15 +168,13 @@ def run_evaluation(num_samples, progress=gr.Progress()):
     
     progress(0.1, desc="Loading CUAD dataset...")
     
-    # Load dataset - use QA format version (JSON, no PDFs)
+    # Load dataset
     try:
-        # Try the QA-specific version first (much faster, JSON format)
         dataset = load_dataset("theatticusproject/cuad-qa", trust_remote_code=True, token=hf_token)
         test_data = dataset["test"]
         print(f"✓ Loaded CUAD-QA dataset with {len(test_data)} samples")
     except Exception as e:
         try:
-            # Fallback to original but limit to avoid PDF downloads
             dataset = load_dataset("cuad", split="test[:1000]", trust_remote_code=True, token=hf_token)
             test_data = dataset
             print(f"✓ Loaded CUAD dataset with {len(test_data)} samples")
@@ -109,97 +187,140 @@ def run_evaluation(num_samples, progress=gr.Progress()):
     
     progress(0.2, desc=f"Starting evaluation on {num_samples} samples...")
     
-    # Initialize metrics
-    exact_matches = []
-    f1_scores = []
-    predictions = []
+    # Initialize storage for predictions and ground truths
+    all_top_k_predictions = []
+    all_ground_truths = []
+    all_has_answer_flags = []
+    
+    # Storage for detailed results
+    detailed_results = []
     
     # Run evaluation
     for i, example in enumerate(test_subset):
-        progress((0.2 + 0.7 * i / num_samples), desc=f"Processing sample {i+1}/{num_samples}")
+        progress((0.2 + 0.6 * i / num_samples), desc=f"Processing sample {i+1}/{num_samples}")
         
         try:
             context = example["context"]
             question = example["question"]
             answers = example["answers"]
             
-            # Get model prediction
-            result = qa_pipeline(question=question, context=context)
-            predicted_answer = result["answer"]
+            # Check if question has answers
+            has_ans = has_answer(answers)
+            all_has_answer_flags.append(has_ans)
+            all_ground_truths.append(answers)
+            
+            # Get top-3 predictions
+            top_k_preds = get_top_k_predictions(qa_pipeline, question, context, k=3)
+            all_top_k_predictions.append(top_k_preds)
             
-            # Get ground truth answers
-            if answers["text"] and len(answers["text"]) > 0:
+            # Get ground truth for display
+            if has_ans:
                 ground_truth = answers["text"][0] if isinstance(answers["text"], list) else answers["text"]
             else:
-                ground_truth = ""
+                ground_truth = "[No Answer]"
             
-            # Calculate metrics
-            em = exact_match_score(predicted_answer, ground_truth)
-            f1 = f1_score_qa(predicted_answer, ground_truth)
-            
-            exact_matches.append(em)
-            f1_scores.append(f1)
+            # Calculate metrics for this sample
+            if has_ans and top_k_preds:
+                top1_f1 = f1_score_qa(top_k_preds[0]["answer"], ground_truth)
+                top3_f1 = max([f1_score_qa(pred["answer"], ground_truth) for pred in top_k_preds[:3]])
+                em = exact_match_score(top_k_preds[0]["answer"], ground_truth)
+            else:
+                top1_f1 = 0
+                top3_f1 = 0
+                em = 0
             
-            predictions.append({
+            detailed_results.append({
                 "Sample_ID": i+1,
                 "Question": question[:100] + "..." if len(question) > 100 else question,
-                "Predicted_Answer": predicted_answer,
+                "Has_Answer": has_ans,
+                "Top1_Prediction": top_k_preds[0]["answer"] if top_k_preds else "[No Prediction]",
+                "Top3_Predictions": " | ".join([p["answer"] for p in top_k_preds[:3]]),
                 "Ground_Truth": ground_truth,
+                "Top1_F1": round(top1_f1, 3),
+                "Top3_F1": round(top3_f1, 3),
                 "Exact_Match": em,
-                "F1_Score": round(f1, 3),
-                "Confidence": round(result["score"], 3)
+                "Top1_Confidence": round(top_k_preds[0]["score"], 3) if top_k_preds else 0
             })
             
         except Exception as e:
             print(f"Error processing sample {i}: {e}")
             continue
     
-    progress(0.9, desc="Calculating final metrics...")
+    progress(0.8, desc="Calculating final metrics...")
     
-    # Calculate final metrics
-    if len(exact_matches) == 0:
-        return "❌ No samples were successfully processed", pd.DataFrame(), None
+    # Filter for questions with answers only
+    has_ans_predictions = [pred for pred, has_ans in zip(all_top_k_predictions, all_has_answer_flags) if has_ans]
+    has_ans_ground_truths = [gt for gt, has_ans in zip(all_ground_truths, all_has_answer_flags) if has_ans]
     
-    avg_exact_match = np.mean(exact_matches) * 100
-    avg_f1_score = np.mean(f1_scores) * 100
+    if len(has_ans_predictions) == 0:
+        return "❌ No samples with answers were found", pd.DataFrame(), None
+    
+    # Calculate Top-K Has Answer F1 scores
+    top1_has_ans_f1 = calculate_top_k_has_ans_f1(has_ans_predictions, has_ans_ground_truths, k=1) * 100
+    top3_has_ans_f1 = calculate_top_k_has_ans_f1(has_ans_predictions, has_ans_ground_truths, k=3) * 100
+    
+    # Calculate overall metrics
+    total_samples = len(detailed_results)
+    has_answer_samples = len(has_ans_predictions)
+    avg_exact_match = np.mean([r["Exact_Match"] for r in detailed_results]) * 100
+    avg_top1_f1 = np.mean([r["Top1_F1"] for r in detailed_results if r["Has_Answer"]]) * 100
+    avg_top3_f1 = np.mean([r["Top3_F1"] for r in detailed_results if r["Has_Answer"]]) * 100
     
     # Create results summary
     results_summary = f"""
 # 📊 CUAD Model Evaluation Results
-## 🎯 Overall Performance
+
+## 🎯 Model Performance
 - **Model**: AvocadoMuffin/roberta-cuad-qa-v3
 - **Dataset**: CUAD (Contract Understanding Atticus Dataset)
-- **Samples Evaluated**: {len(exact_matches)}
+- **Total Samples**: {total_samples}
+- **Samples with Answers**: {has_answer_samples}
 - **Evaluation Date**: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}
-## 📈 Metrics
+
+## 📈 Key Metrics (Industry Standard)
+- **Top 1 Has Ans F1**: {top1_has_ans_f1:.2f}%
+- **Top 3 Has Ans F1**: {top3_has_ans_f1:.2f}%
+
+## 📋 Additional Metrics
 - **Exact Match Score**: {avg_exact_match:.2f}%
-- **F1 Score**: {avg_f1_score:.2f}%
-## 🔍 Performance Analysis
-- **High Confidence Predictions**: {len([p for p in predictions if p['Confidence'] > 0.8])} ({len([p for p in predictions if p['Confidence'] > 0.8])/len(predictions)*100:.1f}%)
-- **Perfect Matches**: {len([p for p in predictions if p['Exact_Match'] == 1])} ({len([p for p in predictions if p['Exact_Match'] == 1])/len(predictions)*100:.1f}%)
-- **High F1 Scores (>0.8)**: {len([p for p in predictions if p['F1_Score'] > 0.8])} ({len([p for p in predictions if p['F1_Score'] > 0.8])/len(predictions)*100:.1f}%)
+- **Average Top-1 F1**: {avg_top1_f1:.2f}%
+- **Average Top-3 F1**: {avg_top3_f1:.2f}%
+
+## 🔍 Performance Breakdown
+- **High Confidence Predictions (>0.8)**: {len([r for r in detailed_results if r['Top1_Confidence'] > 0.8])} ({len([r for r in detailed_results if r['Top1_Confidence'] > 0.8])/total_samples*100:.1f}%)
+- **Perfect Matches**: {len([r for r in detailed_results if r['Exact_Match'] == 1])} ({len([r for r in detailed_results if r['Exact_Match'] == 1])/total_samples*100:.1f}%)
+- **High F1 Scores (>0.8)**: {len([r for r in detailed_results if r['Top1_F1'] > 0.8])} ({len([r for r in detailed_results if r['Top1_F1'] > 0.8])/has_answer_samples*100:.1f}%)
+
+## 📊 Comparison with Benchmarks
+Your model's **Top 1 Has Ans F1** of {top1_has_ans_f1:.2f}% can be compared to:
+- gustavhartz/roberta-base-cuad-finetuned: 85.68%
+- Rakib/roberta-base-on-cuad: 81.26%
 """
     
     # Create detailed results DataFrame
-    df = pd.DataFrame(predictions)
+    df = pd.DataFrame(detailed_results)
     
     # Save results to file
     timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
     results_file = f"cuad_evaluation_results_{timestamp}.json"
     
-    detailed_results = {
+    complete_results = {
         "model_name": "AvocadoMuffin/roberta-cuad-qa-v3",
         "dataset": "cuad",
-        "num_samples": len(exact_matches),
+        "total_samples": total_samples,
+        "has_answer_samples": has_answer_samples,
+        "top1_has_ans_f1": top1_has_ans_f1,
+        "top3_has_ans_f1": top3_has_ans_f1,
         "exact_match_score": avg_exact_match,
-        "f1_score": avg_f1_score,
+        "avg_top1_f1": avg_top1_f1,
+        "avg_top3_f1": avg_top3_f1,
         "evaluation_date": datetime.now().isoformat(),
-        "predictions": predictions
+        "detailed_results": detailed_results
     }
     
     try:
         with open(results_file, "w") as f:
-            json.dump(detailed_results, f, indent=2)
+            json.dump(complete_results, f, indent=2)
         print(f"✓ Results saved to {results_file}")
     except Exception as e:
         print(f"⚠ Warning: Could not save results file: {e}")
@@ -217,7 +338,8 @@ def create_gradio_interface():
         <div style="text-align: center; padding: 20px;">
             <h1>🏛️ CUAD Model Evaluation Dashboard</h1>
             <p>Evaluate your CUAD (Contract Understanding Atticus Dataset) Question Answering model</p>
-            <p><strong>Model:</strong> AvocadoMuffin/roberta-cuad-qa-v2</p>
+            <p><strong>Model:</strong> AvocadoMuffin/roberta-cuad-qa-v3</p>
+            <p><em>Now with industry-standard Top-K Has Answer F1 metrics!</em></p>
         </div>
         """)
         
@@ -242,12 +364,14 @@ def create_gradio_interface():
                 
                 gr.HTML("""
                 <div style="margin-top: 20px; padding: 15px; background-color: #f0f0f0; border-radius: 8px;">
-                    <h4>📋 What this evaluates:</h4>
+                    <h4>📋 Evaluation Metrics:</h4>
                     <ul>
+                        <li><strong>Top 1 Has Ans F1</strong>: F1 score for single best answer (industry standard)</li>
+                        <li><strong>Top 3 Has Ans F1</strong>: F1 score allowing up to 3 predictions</li>
                         <li><strong>Exact Match</strong>: Percentage of perfect predictions</li>
-                        <li><strong>F1 Score</strong>: Token-level overlap between prediction and ground truth</li>
-                        <li><strong>Confidence</strong>: Model's confidence in its predictions</li>
+                        <li><strong>Confidence</strong>: Model's confidence in predictions</li>
                     </ul>
+                    <p><em>Note: "Has Ans" metrics only consider questions that have valid answers.</em></p>
                 </div>
                 """)
             
@@ -297,14 +421,15 @@ def create_gradio_interface():
         <div style="text-align: center; margin-top: 30px; padding: 20px; color: #666;">
             <p>🤖 Powered by Hugging Face Transformers & Gradio</p>
             <p>📚 CUAD Dataset by The Atticus Project</p>
+            <p>📊 Now with industry-standard Top-K Has Answer F1 metrics</p>
         </div>
         """)
     
     return demo
     
 if __name__ == "__main__":
-    print("CUAD Model Evaluation with Gradio Interface")
-    print("=" * 50)
+    print("CUAD Model Evaluation with Top-K Has Answer F1 Metrics")
+    print("=" * 60)
     
     # Check if CUDA is available
     if torch.cuda.is_available():