Spaces:
Running
Running
| """Assertion-completion scoring, identical to RepoPeftBench's evaluation/metrics.py | |
| (exact match and edit similarity; CodeBLEU omitted).""" | |
| from difflib import SequenceMatcher | |
| FIM_TOKENS = ("<|fim_prefix|>", "<|fim_suffix|>", "<|fim_middle|>") | |
| def strip_fim_tokens(s: str) -> str: | |
| for tok in FIM_TOKENS: | |
| s = s.replace(tok, "") | |
| return s.strip() | |
| def strip_comments(s: str) -> str: | |
| return s.split("#")[0].strip() | |
| def normalize_for_match(s: str) -> str: | |
| s = s.strip().rstrip(":, \t") | |
| s = s.replace(", ", ",").replace(" ,", ",") | |
| return " ".join(s.split()) | |
| def exact_match(pred: str, ref: str) -> bool: | |
| return normalize_for_match(pred) == normalize_for_match(ref) | |
| def edit_similarity(pred: str, ref: str) -> float: | |
| return SequenceMatcher(None, pred, ref).ratio() | |
| def postprocess_prediction(pred: str, target: str) -> str: | |
| pred = strip_fim_tokens(pred) | |
| if "\n" not in target and "\n" in pred: | |
| pred = pred.split("\n")[0] | |
| pred = strip_comments(pred) | |
| ref = strip_comments(target).strip() | |
| if ref: | |
| if "," not in ref and "," in pred: | |
| pred = pred.split(",")[0].strip() | |
| if len(ref.split()) == 1 and len(pred.strip().split()) > 1: | |
| pred = pred.strip().split()[0] | |
| return pred.strip() | |
| def compute_metrics(pred: str, ref: str) -> dict: | |
| pred_clean = postprocess_prediction(pred, ref) | |
| ref_clean = strip_comments(ref) | |
| return { | |
| "pred_clean": pred_clean, | |
| "exact_match": exact_match(pred_clean, ref_clean), | |
| "edit_similarity": edit_similarity(pred_clean, ref_clean), | |
| } | |