Update app.py

app.py

@@ -2,7 +2,7 @@ import os
 import json
 import numpy as np
 from datasets import load_dataset
-from transformers import AutoTokenizer, AutoModelForQuestionAnswering, pipeline
+from transformers import AutoTokenizer, AutoModelForQuestionAnswering
 import torch
 from sklearn.metrics import f1_score
 import re
@@ -12,311 +12,143 @@ from huggingface_hub import login
 import gradio as gr
 import pandas as pd
 from datetime import datetime
+import matplotlib.pyplot as plt
 
+# Normalization functions (same as extractor)
 def normalize_answer(s):
-    """Normalize answer for evaluation"""
     def remove_articles(text):
         return re.sub(r'\b(a|an|the)\b', ' ', text)
-    
     def white_space_fix(text):
         return ' '.join(text.split())
-    
     def remove_punc(text):
         exclude = set(string.punctuation)
         return ''.join(ch for ch in text if ch not in exclude)
-    
     def lower(text):
         return text.lower()
-    
     return white_space_fix(remove_articles(remove_punc(lower(s))))
 
 def f1_score_qa(prediction, ground_truth):
-    """Calculate F1 score for QA"""
     prediction_tokens = normalize_answer(prediction).split()
     ground_truth_tokens = normalize_answer(ground_truth).split()
-    
-    if len(prediction_tokens) == 0 or len(ground_truth_tokens) == 0:
-        return int(prediction_tokens == ground_truth_tokens)
-    
     common = Counter(prediction_tokens) & Counter(ground_truth_tokens)
     num_same = sum(common.values())
-    
     if num_same == 0:
         return 0
-    
     precision = 1.0 * num_same / len(prediction_tokens)
     recall = 1.0 * num_same / len(ground_truth_tokens)
-    f1 = (2 * precision * recall) / (precision + recall)
-    return f1
+    return (2 * precision * recall) / (precision + recall)
 
 def exact_match_score(prediction, ground_truth):
-    """Calculate exact match score"""
     return normalize_answer(prediction) == normalize_answer(ground_truth)
 
-def evaluate_model():
-    # Authenticate with Hugging Face using the token
-    hf_token = os.getenv("EVAL_TOKEN")
-    if hf_token:
-        try:
-            login(token=hf_token)
-            print("✓ Authenticated with Hugging Face")
-        except Exception as e:
-            print(f"⚠ Warning: Could not authenticate with HF token: {e}")
-    else:
-        print("⚠ Warning: EVAL_TOKEN not found in environment variables")
-    
-    print("Loading model and tokenizer...")
-    model_name = "AvocadoMuffin/roberta-cuad-qa-v2"
+# Identical confidence calculation to extractor
+def calculate_confidence(model, tokenizer, question, context):
+    inputs = tokenizer(
+        question, 
+        context, 
+        return_tensors="pt",
+        truncation=True,
+        max_length=512,
+        stride=128,
+        padding=True
+    )
     
-    try:
-        tokenizer = AutoTokenizer.from_pretrained(model_name, token=hf_token)
-        model = AutoModelForQuestionAnswering.from_pretrained(model_name, token=hf_token)
-        qa_pipeline = pipeline("question-answering", model=model, tokenizer=tokenizer)
-        print("✓ Model loaded successfully")
-        return qa_pipeline, hf_token
-    except Exception as e:
-        print(f"✗ Error loading model: {e}")
-        return None, None
+    if torch.cuda.is_available():
+        inputs = {k: v.cuda() for k, v in inputs.items()}
+        model = model.cuda()
 
-def run_evaluation(num_samples, progress=gr.Progress()):
-    """Run evaluation and return results for Gradio interface"""
-    
-    # Load model
-    qa_pipeline, hf_token = evaluate_model()
-    if qa_pipeline is None:
-        return "❌ Failed to load model", pd.DataFrame(), None
-    
-    progress(0.1, desc="Loading CUAD dataset...")
-    
-    # Load dataset - use QA format version (JSON, no PDFs)
-    try:
-        # Try the QA-specific version first (much faster, JSON format)
-        dataset = load_dataset("theatticusproject/cuad-qa", trust_remote_code=True, token=hf_token)
-        test_data = dataset["test"]
-        print(f"✓ Loaded CUAD-QA dataset with {len(test_data)} samples")
-    except Exception as e:
-        try:
-            # Fallback to original but limit to avoid PDF downloads
-            dataset = load_dataset("cuad", split="test[:1000]", trust_remote_code=True, token=hf_token)
-            test_data = dataset
-            print(f"✓ Loaded CUAD dataset with {len(test_data)} samples")
-        except Exception as e2:
-            return f"❌ Error loading dataset: {e2}", pd.DataFrame(), None
+    with torch.no_grad():
+        outputs = model(**inputs)
+
+    start_probs = torch.softmax(outputs.start_logits, dim=1)
+    end_probs = torch.softmax(outputs.end_logits, dim=1)
+    answer_start = torch.argmax(outputs.start_logits)
+    answer_end = torch.argmax(outputs.end_logits) + 1
     
-    # Limit samples
-    num_samples = min(num_samples, len(test_data))
-    test_subset = test_data.select(range(num_samples))
+    start_prob = start_probs[0, answer_start].item()
+    end_prob = end_probs[0, answer_end-1].item()
+    confidence = np.sqrt(start_prob * end_prob)
     
-    progress(0.2, desc=f"Starting evaluation on {num_samples} samples...")
+    answer_tokens = inputs["input_ids"][0][answer_start:answer_end]
+    answer = tokenizer.decode(answer_tokens, skip_special_tokens=True).strip()
     
-    # Initialize metrics
-    exact_matches = []
-    f1_scores = []
-    predictions = []
+    return answer, float(confidence)
+
+def run_evaluation(num_samples=100):
+    # Authenticate
+    if token := os.getenv("HF_TOKEN"):
+        login(token=token)
     
-    # Run evaluation
-    for i, example in enumerate(test_subset):
-        progress((0.2 + 0.7 * i / num_samples), desc=f"Processing sample {i+1}/{num_samples}")
+    # Load model same as extractor
+    model_name = "AvocadoMuffin/roberta-cuad-qa-v2"
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    model = AutoModelForQuestionAnswering.from_pretrained(model_name)
+    
+    # Load CUAD dataset
+    dataset = load_dataset("theatticusproject/cuad-qa", token=token)
+    test_data = dataset["test"].select(range(min(num_samples, len(dataset["test"]))))
+    
+    results = []
+    for example in test_data:
+        context = example["context"]
+        question = example["question"]
+        gt_answer = example["answers"]["text"][0] if example["answers"]["text"] else ""
         
-        try:
-            context = example["context"]
-            question = example["question"]
-            answers = example["answers"]
-            
-            # Get model prediction
-            result = qa_pipeline(question=question, context=context)
-            predicted_answer = result["answer"]
-            
-            # Get ground truth answers
-            if answers["text"] and len(answers["text"]) > 0:
-                ground_truth = answers["text"][0] if isinstance(answers["text"], list) else answers["text"]
-            else:
-                ground_truth = ""
-            
-            # Calculate metrics
-            em = exact_match_score(predicted_answer, ground_truth)
-            f1 = f1_score_qa(predicted_answer, ground_truth)
-            
-            exact_matches.append(em)
-            f1_scores.append(f1)
-            
-            predictions.append({
-                "Sample_ID": i+1,
-                "Question": question[:100] + "..." if len(question) > 100 else question,
-                "Predicted_Answer": predicted_answer,
-                "Ground_Truth": ground_truth,
-                "Exact_Match": em,
-                "F1_Score": round(f1, 3),
-                "Confidence": round(result["score"], 3)
-            })
-            
-        except Exception as e:
-            print(f"Error processing sample {i}: {e}")
-            continue
-    
-    progress(0.9, desc="Calculating final metrics...")
+        pred_answer, confidence = calculate_confidence(model, tokenizer, question, context)
+        
+        results.append({
+            "question": question,
+            "prediction": pred_answer,
+            "ground_truth": gt_answer,
+            "confidence": confidence,
+            "exact_match": exact_match_score(pred_answer, gt_answer),
+            "f1": f1_score_qa(pred_answer, gt_answer)
+        })
+    
+    # Generate report
+    df = pd.DataFrame(results)
+    avg_metrics = {
+        "exact_match": df["exact_match"].mean() * 100,
+        "f1": df["f1"].mean() * 100,
+        "confidence": df["confidence"].mean() * 100
+    }
     
-    # Calculate final metrics
-    if len(exact_matches) == 0:
-        return "❌ No samples were successfully processed", pd.DataFrame(), None
+    # Confidence calibration analysis
+    high_conf_correct = df[(df["confidence"] > 0.8) & (df["exact_match"] == 1)].shape[0]
+    high_conf_total = df[df["confidence"] > 0.8].shape[0]
     
-    avg_exact_match = np.mean(exact_matches) * 100
-    avg_f1_score = np.mean(f1_scores) * 100
+    report = f"""
+    CUAD Evaluation Report (n={len(df)})
+    ========================
+    Accuracy:
+    - Exact Match: {avg_metrics['exact_match']:.2f}%
+    - F1 Score: {avg_metrics['f1']:.2f}%
     
-    # Create results summary
-    results_summary = f"""
-# 📊 CUAD Model Evaluation Results
-## 🎯 Overall Performance
-- **Model**: AvocadoMuffin/roberta-cuad-qa-v3
-- **Dataset**: CUAD (Contract Understanding Atticus Dataset)
-- **Samples Evaluated**: {len(exact_matches)}
-- **Evaluation Date**: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}
-## 📈 Metrics
-- **Exact Match Score**: {avg_exact_match:.2f}%
-- **F1 Score**: {avg_f1_score:.2f}%
-## 🔍 Performance Analysis
-- **High Confidence Predictions**: {len([p for p in predictions if p['Confidence'] > 0.8])} ({len([p for p in predictions if p['Confidence'] > 0.8])/len(predictions)*100:.1f}%)
-- **Perfect Matches**: {len([p for p in predictions if p['Exact_Match'] == 1])} ({len([p for p in predictions if p['Exact_Match'] == 1])/len(predictions)*100:.1f}%)
-- **High F1 Scores (>0.8)**: {len([p for p in predictions if p['F1_Score'] > 0.8])} ({len([p for p in predictions if p['F1_Score'] > 0.8])/len(predictions)*100:.1f}%)
-"""
+    Confidence Analysis:
+    - Avg Confidence: {avg_metrics['confidence']:.2f}%
+    - High-Confidence (>80%) Accuracy: {high_conf_correct}/{high_conf_total} ({high_conf_correct/max(1,high_conf_total)*100:.1f}%)
     
-    # Create detailed results DataFrame
-    df = pd.DataFrame(predictions)
+    Confidence vs Accuracy:
+    {df[['confidence', 'exact_match']].corr().iloc[0,1]:.3f} correlation
+    """
     
-    # Save results to file
+    # Save results
     timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-    results_file = f"cuad_evaluation_results_{timestamp}.json"
-    
-    detailed_results = {
-        "model_name": "AvocadoMuffin/roberta-cuad-qa-v3",
-        "dataset": "cuad",
-        "num_samples": len(exact_matches),
-        "exact_match_score": avg_exact_match,
-        "f1_score": avg_f1_score,
-        "evaluation_date": datetime.now().isoformat(),
-        "predictions": predictions
-    }
-    
-    try:
-        with open(results_file, "w") as f:
-            json.dump(detailed_results, f, indent=2)
-        print(f"✓ Results saved to {results_file}")
-    except Exception as e:
-        print(f"⚠ Warning: Could not save results file: {e}")
-        results_file = None
-    
-    progress(1.0, desc="✅ Evaluation completed!")
-    
-    return results_summary, df, results_file
+    results_file = f"cuad_eval_{timestamp}.json"
+    with open(results_file, "w") as f:
+        json.dump({
+            "metrics": avg_metrics,
+            "samples": results,
+            "config": {
+                "model": model_name,
+                "confidence_method": "geometric_mean_start_end_probs"
+            }
+        }, f, indent=2)
+    
+    return report, df, results_file
 
-def create_gradio_interface():
-    """Create Gradio interface for CUAD evaluation"""
-    
-    with gr.Blocks(title="CUAD Model Evaluator", theme=gr.themes.Soft()) as demo:
-        gr.HTML("""
-        <div style="text-align: center; padding: 20px;">
-            <h1>🏛️ CUAD Model Evaluation Dashboard</h1>
-            <p>Evaluate your CUAD (Contract Understanding Atticus Dataset) Question Answering model</p>
-            <p><strong>Model:</strong> AvocadoMuffin/roberta-cuad-qa-v2</p>
-        </div>
-        """)
-        
-        with gr.Row():
-            with gr.Column(scale=1):
-                gr.HTML("<h3>⚙️ Evaluation Settings</h3>")
-                
-                num_samples = gr.Slider(
-                    minimum=10,
-                    maximum=500,
-                    value=100,
-                    step=10,
-                    label="Number of samples to evaluate",
-                    info="Choose between 10-500 samples (more samples = more accurate but slower)"
-                )
-                
-                evaluate_btn = gr.Button(
-                    "🚀 Start Evaluation", 
-                    variant="primary",
-                    size="lg"
-                )
-                
-                gr.HTML("""
-                <div style="margin-top: 20px; padding: 15px; background-color: #f0f0f0; border-radius: 8px;">
-                    <h4>📋 What this evaluates:</h4>
-                    <ul>
-                        <li><strong>Exact Match</strong>: Percentage of perfect predictions</li>
-                        <li><strong>F1 Score</strong>: Token-level overlap between prediction and ground truth</li>
-                        <li><strong>Confidence</strong>: Model's confidence in its predictions</li>
-                    </ul>
-                </div>
-                """)
-            
-            with gr.Column(scale=2):
-                gr.HTML("<h3>📊 Results</h3>")
-                
-                results_summary = gr.Markdown(
-                    value="Click '🚀 Start Evaluation' to begin...",
-                    label="Evaluation Summary"
-                )
-        
-        gr.HTML("<hr>")
-        
-        with gr.Row():
-            gr.HTML("<h3>📋 Detailed Results</h3>")
-        
-        with gr.Row():
-            detailed_results = gr.Dataframe(
-                label="Sample-by-Sample Results",
-                interactive=False,
-                wrap=True
-            )
-        
-        with gr.Row():
-            download_file = gr.File(
-                label="📥 Download Complete Results (JSON)",
-                visible=False
-            )
-        
-        # Event handlers
-        def handle_evaluation(num_samples):
-            summary, df, file_path = run_evaluation(num_samples)
-            if file_path and os.path.exists(file_path):
-                return summary, df, gr.update(visible=True, value=file_path)
-            else:
-                return summary, df, gr.update(visible=False)
-        
-        evaluate_btn.click(
-            fn=handle_evaluation,
-            inputs=[num_samples],
-            outputs=[results_summary, detailed_results, download_file],
-            show_progress=True
-        )
-        
-        # Footer
-        gr.HTML("""
-        <div style="text-align: center; margin-top: 30px; padding: 20px; color: #666;">
-            <p>�� Powered by Hugging Face Transformers & Gradio</p>
-            <p>📚 CUAD Dataset by The Atticus Project</p>
-        </div>
-        """)
-    
-    return demo
-    
 if __name__ == "__main__":
-    print("CUAD Model Evaluation with Gradio Interface")
-    print("=" * 50)
-    
-    # Check if CUDA is available
-    if torch.cuda.is_available():
-        print(f"✓ CUDA available: {torch.cuda.get_device_name(0)}")
-    else:
-        print("! Running on CPU")
-    
-    # Create and launch Gradio interface
-    demo = create_gradio_interface()
-    demo.launch(
-        server_name="0.0.0.0",
-        server_port=7860,
-        share=True,
-        debug=True
-    )
+    report, df, _ = run_evaluation()
+    print(report)
+    print("\nSample predictions:")
+    print(df.head())