modules/utils/examples.py

"""Preset examples for metric demonstrations."""

from typing import List, Dict, Tuple


class EvaluationExample:
    """Represents a ground truth / prediction pair example."""
    
    def __init__(self, name: str, ground_truth: str, prediction: str, 
                 description: str = "", tags: List[str] = None):
        self.name = name
        self.ground_truth = ground_truth
        self.prediction = prediction
        self.description = description
        self.tags = tags or []


# Text comparison examples
TEXT_EXAMPLES = [
    EvaluationExample(
        name="Exact Match",
        ground_truth="The cat sat on the mat",
        prediction="The cat sat on the mat",
        description="Perfect match - all metrics should be 1.0",
        tags=["exact", "perfect"]
    ),
    EvaluationExample(
        name="Paraphrase (Synonyms)",
        ground_truth="The cat sat on the mat",
        prediction="The cat was sitting on the mat",
        description="Same meaning, different words - good BERT Score, lower BLEU/ROGUE",
        tags=["paraphrase", "semantic"]
    ),
    EvaluationExample(
        name="Partial Match",
        ground_truth="The cat sat on the mat and looked outside",
        prediction="The cat sat on the mat",
        description="Incomplete prediction - brevity penalty applies",
        tags=["partial", "incomplete"]
    ),
    EvaluationExample(
        name="Wrong Answer",
        ground_truth="Paris is the capital of France",
        prediction="Berlin is the capital of France",
        description="Completely wrong but similar structure",
        tags=["wrong", "incorrect"]
    ),
    EvaluationExample(
        name="Extra Content",
        ground_truth="The cat sat on the mat",
        prediction="The cat sat on the mat and then jumped off quickly",
        description="Extra words added - recall will suffer",
        tags=["extra", "verbose"]
    ),
    EvaluationExample(
        name="Word Order Changed",
        ground_truth="The cat sat on the mat",
        prediction="On the mat sat the cat",
        description="Same words, different order - ROGUE-L should handle this",
        tags=["reorder", "scrambled"]
    ),
    EvaluationExample(
        name="Long Text - Translation",
        ground_truth="Machine translation has revolutionized how we communicate across languages, enabling instant understanding of foreign texts.",
        prediction="Machine translation has changed how we communicate between languages, allowing instant comprehension of foreign texts.",
        description="Realistic translation scenario",
        tags=["translation", "long"]
    ),
    EvaluationExample(
        name="Summarization - Key Points",
        ground_truth="The research paper demonstrates that neural networks can effectively predict protein structures with high accuracy, potentially revolutionizing drug discovery and biological research.",
        prediction="Neural networks can predict protein structures accurately, which could transform drug discovery.",
        description="Abstractive summarization - ROGUE should capture key points",
        tags=["summarization", "abstractive"]
    ),
]

# MRR-specific examples (question-answering format)
MRR_EXAMPLES = [
    {
        "name": "Correct at Position 1",
        "question": "What is the capital of France?",
        "ranked_answers": ["Paris", "London", "Berlin", "Madrid"],
        "correct_answer": "Paris",
        "description": "Perfect ranking - MRR = 1.0"
    },
    {
        "name": "Correct at Position 2",
        "question": "What is the capital of France?",
        "ranked_answers": ["London", "Paris", "Berlin", "Madrid"],
        "correct_answer": "Paris",
        "description": "Good ranking - MRR = 0.5"
    },
    {
        "name": "Correct at Position 3",
        "question": "What is the capital of France?",
        "ranked_answers": ["London", "Berlin", "Paris", "Madrid"],
        "correct_answer": "Paris",
        "description": "Average ranking - MRR = 0.33"
    },
    {
        "name": "Not in Top 3",
        "question": "What is the capital of France?",
        "ranked_answers": ["London", "Berlin", "Madrid", "Paris"],
        "correct_answer": "Paris",
        "description": "Poor ranking - MRR = 0.25"
    },
    {
        "name": "Multiple Questions Batch",
        "questions": [
            {"q": "Capital of France?", "ranked": ["Paris", "London", "Berlin"], "correct": "Paris"},
            {"q": "Capital of Japan?", "ranked": ["Beijing", "Tokyo", "Seoul"], "correct": "Tokyo"},
            {"q": "Capital of UK?", "ranked": ["London", "Paris", "Berlin"], "correct": "London"},
        ],
        "description": "Batch evaluation - calculates mean across multiple questions"
    }
]


def get_text_example_names() -> List[str]:
    """Get list of available text example names."""
    return [ex.name for ex in TEXT_EXAMPLES]


def get_text_example(name: str) -> EvaluationExample:
    """Get a specific text example by name.
    
    Args:
        name: Example name
        
    Returns:
        EvaluationExample object
    """
    for ex in TEXT_EXAMPLES:
        if ex.name == name:
            return ex
    return TEXT_EXAMPLES[0]


def get_mrr_example_names() -> List[str]:
    """Get list of available MRR example names."""
    return [ex["name"] for ex in MRR_EXAMPLES]


def get_mrr_example(name: str) -> Dict:
    """Get a specific MRR example by name."""
    for ex in MRR_EXAMPLES:
        if ex["name"] == name:
            return ex
    return MRR_EXAMPLES[0]