Spaces:
Runtime error
Runtime error
File size: 5,545 Bytes
99f6668 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 | """Preset examples for metric demonstrations."""
from typing import List, Dict, Tuple
class EvaluationExample:
"""Represents a ground truth / prediction pair example."""
def __init__(self, name: str, ground_truth: str, prediction: str,
description: str = "", tags: List[str] = None):
self.name = name
self.ground_truth = ground_truth
self.prediction = prediction
self.description = description
self.tags = tags or []
# Text comparison examples
TEXT_EXAMPLES = [
EvaluationExample(
name="Exact Match",
ground_truth="The cat sat on the mat",
prediction="The cat sat on the mat",
description="Perfect match - all metrics should be 1.0",
tags=["exact", "perfect"]
),
EvaluationExample(
name="Paraphrase (Synonyms)",
ground_truth="The cat sat on the mat",
prediction="The cat was sitting on the mat",
description="Same meaning, different words - good BERT Score, lower BLEU/ROGUE",
tags=["paraphrase", "semantic"]
),
EvaluationExample(
name="Partial Match",
ground_truth="The cat sat on the mat and looked outside",
prediction="The cat sat on the mat",
description="Incomplete prediction - brevity penalty applies",
tags=["partial", "incomplete"]
),
EvaluationExample(
name="Wrong Answer",
ground_truth="Paris is the capital of France",
prediction="Berlin is the capital of France",
description="Completely wrong but similar structure",
tags=["wrong", "incorrect"]
),
EvaluationExample(
name="Extra Content",
ground_truth="The cat sat on the mat",
prediction="The cat sat on the mat and then jumped off quickly",
description="Extra words added - recall will suffer",
tags=["extra", "verbose"]
),
EvaluationExample(
name="Word Order Changed",
ground_truth="The cat sat on the mat",
prediction="On the mat sat the cat",
description="Same words, different order - ROGUE-L should handle this",
tags=["reorder", "scrambled"]
),
EvaluationExample(
name="Long Text - Translation",
ground_truth="Machine translation has revolutionized how we communicate across languages, enabling instant understanding of foreign texts.",
prediction="Machine translation has changed how we communicate between languages, allowing instant comprehension of foreign texts.",
description="Realistic translation scenario",
tags=["translation", "long"]
),
EvaluationExample(
name="Summarization - Key Points",
ground_truth="The research paper demonstrates that neural networks can effectively predict protein structures with high accuracy, potentially revolutionizing drug discovery and biological research.",
prediction="Neural networks can predict protein structures accurately, which could transform drug discovery.",
description="Abstractive summarization - ROGUE should capture key points",
tags=["summarization", "abstractive"]
),
]
# MRR-specific examples (question-answering format)
MRR_EXAMPLES = [
{
"name": "Correct at Position 1",
"question": "What is the capital of France?",
"ranked_answers": ["Paris", "London", "Berlin", "Madrid"],
"correct_answer": "Paris",
"description": "Perfect ranking - MRR = 1.0"
},
{
"name": "Correct at Position 2",
"question": "What is the capital of France?",
"ranked_answers": ["London", "Paris", "Berlin", "Madrid"],
"correct_answer": "Paris",
"description": "Good ranking - MRR = 0.5"
},
{
"name": "Correct at Position 3",
"question": "What is the capital of France?",
"ranked_answers": ["London", "Berlin", "Paris", "Madrid"],
"correct_answer": "Paris",
"description": "Average ranking - MRR = 0.33"
},
{
"name": "Not in Top 3",
"question": "What is the capital of France?",
"ranked_answers": ["London", "Berlin", "Madrid", "Paris"],
"correct_answer": "Paris",
"description": "Poor ranking - MRR = 0.25"
},
{
"name": "Multiple Questions Batch",
"questions": [
{"q": "Capital of France?", "ranked": ["Paris", "London", "Berlin"], "correct": "Paris"},
{"q": "Capital of Japan?", "ranked": ["Beijing", "Tokyo", "Seoul"], "correct": "Tokyo"},
{"q": "Capital of UK?", "ranked": ["London", "Paris", "Berlin"], "correct": "London"},
],
"description": "Batch evaluation - calculates mean across multiple questions"
}
]
def get_text_example_names() -> List[str]:
"""Get list of available text example names."""
return [ex.name for ex in TEXT_EXAMPLES]
def get_text_example(name: str) -> EvaluationExample:
"""Get a specific text example by name.
Args:
name: Example name
Returns:
EvaluationExample object
"""
for ex in TEXT_EXAMPLES:
if ex.name == name:
return ex
return TEXT_EXAMPLES[0]
def get_mrr_example_names() -> List[str]:
"""Get list of available MRR example names."""
return [ex["name"] for ex in MRR_EXAMPLES]
def get_mrr_example(name: str) -> Dict:
"""Get a specific MRR example by name."""
for ex in MRR_EXAMPLES:
if ex["name"] == name:
return ex
return MRR_EXAMPLES[0]
|