from typing import List, Tuple, Union import re from benchmark.normalization import normalize_text def score_exact_match(prediction: str, expected: str, acceptable_answers: List[str] = None) -> float: pred_norm = normalize_text(prediction) exp_norm = normalize_text(expected) if pred_norm == exp_norm: return 1.0 if acceptable_answers: for acc in acceptable_answers: if pred_norm == normalize_text(acc): return 1.0 # Also support substring matching (e.g., if expected is inside the prediction) if exp_norm in pred_norm: return 1.0 return 0.0 def score_token_f1(prediction: str, expected: str) -> Tuple[float, float, float]: """ Returns (precision, recall, f1) """ pred_tokens = normalize_text(prediction).split() exp_tokens = normalize_text(expected).split() if not pred_tokens or not exp_tokens: if pred_tokens == exp_tokens: return 1.0, 1.0, 1.0 return 0.0, 0.0, 0.0 # Calculate overlaps common = [] temp_exp = list(exp_tokens) for token in pred_tokens: if token in temp_exp: common.append(token) temp_exp.remove(token) num_common = len(common) precision = num_common / len(pred_tokens) recall = num_common / len(exp_tokens) if (precision + recall) == 0: f1 = 0.0 else: f1 = 2 * (precision * recall) / (precision + recall) return precision, recall, f1 def score_numeric(prediction: str, expected: Union[str, float], mode: str = "exact", tolerance: float = 0.0) -> float: """ Extracts first number from prediction and expected, then compares. Modes: "exact", "absolute_tolerance", "relative_tolerance" """ # Helper to find first float in a string def extract_float(s: str) -> Union[float, None]: if isinstance(s, (int, float)): return float(s) match = re.search(r"[-+]?\d*\.\d+|\d+", s) if match: return float(match.group()) return None pred_val = extract_float(prediction) exp_val = extract_float(str(expected)) if pred_val is None or exp_val is None: return 0.0 if mode == "exact": return 1.0 if pred_val == exp_val else 0.0 elif mode == "absolute_tolerance": return 1.0 if abs(pred_val - exp_val) <= tolerance else 0.0 elif mode == "relative_tolerance": if exp_val == 0.0: return 1.0 if pred_val == 0.0 else 0.0 return 1.0 if (abs(pred_val - exp_val) / abs(exp_val)) <= tolerance else 0.0 return 0.0 def score_list(prediction: str, expected_list: List[str]) -> Tuple[float, float, float]: """ Extract list items (separated by comma, semicolon, newline or bullet points) and compare as sets. Returns (precision, recall, f1) """ # Normalize list separator characters to commas pred_cleaned = re.sub(r"[\n;•\-*]", ",", prediction) pred_items = [normalize_text(x) for x in pred_cleaned.split(",") if normalize_text(x)] exp_items = [normalize_text(x) for x in expected_list if normalize_text(x)] if not pred_items or not exp_items: return 0.0, 0.0, 0.0 pred_set = set(pred_items) exp_set = set(exp_items) intersection = pred_set.intersection(exp_set) num_common = len(intersection) precision = num_common / len(pred_set) recall = num_common / len(exp_set) if (precision + recall) == 0: f1 = 0.0 else: f1 = 2 * (precision * recall) / (precision + recall) return precision, recall, f1