| from typing import List, Tuple, Union |
| import re |
| from benchmark.normalization import normalize_text |
|
|
| def score_exact_match(prediction: str, expected: str, acceptable_answers: List[str] = None) -> float: |
| pred_norm = normalize_text(prediction) |
| exp_norm = normalize_text(expected) |
| |
| if pred_norm == exp_norm: |
| return 1.0 |
| |
| if acceptable_answers: |
| for acc in acceptable_answers: |
| if pred_norm == normalize_text(acc): |
| return 1.0 |
| |
| |
| if exp_norm in pred_norm: |
| return 1.0 |
| |
| return 0.0 |
|
|
| def score_token_f1(prediction: str, expected: str) -> Tuple[float, float, float]: |
| """ |
| Returns (precision, recall, f1) |
| """ |
| pred_tokens = normalize_text(prediction).split() |
| exp_tokens = normalize_text(expected).split() |
| |
| if not pred_tokens or not exp_tokens: |
| if pred_tokens == exp_tokens: |
| return 1.0, 1.0, 1.0 |
| return 0.0, 0.0, 0.0 |
| |
| |
| common = [] |
| temp_exp = list(exp_tokens) |
| for token in pred_tokens: |
| if token in temp_exp: |
| common.append(token) |
| temp_exp.remove(token) |
| |
| num_common = len(common) |
| |
| precision = num_common / len(pred_tokens) |
| recall = num_common / len(exp_tokens) |
| |
| if (precision + recall) == 0: |
| f1 = 0.0 |
| else: |
| f1 = 2 * (precision * recall) / (precision + recall) |
| |
| return precision, recall, f1 |
|
|
| def score_numeric(prediction: str, expected: Union[str, float], mode: str = "exact", tolerance: float = 0.0) -> float: |
| """ |
| Extracts first number from prediction and expected, then compares. |
| Modes: "exact", "absolute_tolerance", "relative_tolerance" |
| """ |
| |
| def extract_float(s: str) -> Union[float, None]: |
| if isinstance(s, (int, float)): |
| return float(s) |
| match = re.search(r"[-+]?\d*\.\d+|\d+", s) |
| if match: |
| return float(match.group()) |
| return None |
| |
| pred_val = extract_float(prediction) |
| exp_val = extract_float(str(expected)) |
| |
| if pred_val is None or exp_val is None: |
| return 0.0 |
| |
| if mode == "exact": |
| return 1.0 if pred_val == exp_val else 0.0 |
| |
| elif mode == "absolute_tolerance": |
| return 1.0 if abs(pred_val - exp_val) <= tolerance else 0.0 |
| |
| elif mode == "relative_tolerance": |
| if exp_val == 0.0: |
| return 1.0 if pred_val == 0.0 else 0.0 |
| return 1.0 if (abs(pred_val - exp_val) / abs(exp_val)) <= tolerance else 0.0 |
| |
| return 0.0 |
|
|
| def score_list(prediction: str, expected_list: List[str]) -> Tuple[float, float, float]: |
| """ |
| Extract list items (separated by comma, semicolon, newline or bullet points) |
| and compare as sets. Returns (precision, recall, f1) |
| """ |
| |
| pred_cleaned = re.sub(r"[\n;•\-*]", ",", prediction) |
| pred_items = [normalize_text(x) for x in pred_cleaned.split(",") if normalize_text(x)] |
| exp_items = [normalize_text(x) for x in expected_list if normalize_text(x)] |
| |
| if not pred_items or not exp_items: |
| return 0.0, 0.0, 0.0 |
| |
| pred_set = set(pred_items) |
| exp_set = set(exp_items) |
| |
| intersection = pred_set.intersection(exp_set) |
| num_common = len(intersection) |
| |
| precision = num_common / len(pred_set) |
| recall = num_common / len(exp_set) |
| |
| if (precision + recall) == 0: |
| f1 = 0.0 |
| else: |
| f1 = 2 * (precision * recall) / (precision + recall) |
| |
| return precision, recall, f1 |
|
|
