""" Deterministic graders for the Code Review environment. Rules: - No randomness, no LLM calls — fully deterministic. - All functions return float in [0.0, 1.0]. - grade_identify: partial credit via keyword matching. - grade_fix: strict match first, then token-normalized fallback. """ import re from typing import List # --------------------------------------------------------------------------- # Identify grader — partial scoring # --------------------------------------------------------------------------- def grade_identify(expected_keywords: List[str], response: str) -> float: """ Score an identify action by counting matched keywords. Scoring: - Smooth partial scoring: score = matched / total keywords. - Whole-word regex matching prevents substring false-positives. - No hard threshold — every matched keyword contributes proportionally. - Score is capped at 1.0. Args: expected_keywords: List of keywords the agent should mention. response: The agent's identify explanation (free text). Returns: float in [0.0, 1.0] """ if not response or not expected_keywords: return 0.0 normalized = response.lower() matched = [ kw for kw in expected_keywords if re.search(r"\b" + re.escape(kw.lower()) + r"\b", normalized) ] score = len(matched) / len(expected_keywords) return round(min(score, 1.0), 2) # --------------------------------------------------------------------------- # Fix grader — strict + normalized fallback # --------------------------------------------------------------------------- def _normalize(code: str) -> str: """ Normalize code for comparison: - Strip leading/trailing whitespace - Collapse all internal whitespace runs to a single space - Lowercase """ code = code.strip() code = re.sub(r"\s+", " ", code) return code.lower() def _tokenize(code: str) -> List[str]: """Split normalized code into alphanum tokens for partial overlap scoring.""" return re.findall(r"[a-z0-9]+", _normalize(code)) def grade_fix(expected_code: str, response: str) -> float: """ Score a fix action against the expected fixed code. Scoring tiers: 1. Exact match (after stripping) → 1.0 2. Normalized match (whitespace collapsed) → 0.9 3. Token overlap ≥ 80% → 0.6 4. Token overlap ≥ 50% → 0.3 5. Below 50% overlap → 0.0 The response may contain surrounding explanation — we extract the largest code-like block for comparison before grading. Args: expected_code: The correct fixed version of the buggy code. response: The agent's fix response (may include explanation). Returns: float in [0.0, 1.0] """ if not response: return 0.0 # Tier 1 — exact strip match on raw response before any extraction if response.strip() == expected_code.strip(): return 1.0 candidate = _extract_code(response) # Tier 1b — exact strip match on extracted block if candidate.strip() == expected_code.strip(): return 1.0 # Tier 2 — normalized whitespace match if _normalize(candidate) == _normalize(expected_code): return 0.9 # Tier 3 & 4 — token overlap expected_tokens = _tokenize(expected_code) candidate_tokens = _tokenize(candidate) if not expected_tokens: return 0.0 # Overlap: count how many expected tokens appear in candidate candidate_set = set(candidate_tokens) matched = sum(1 for t in expected_tokens if t in candidate_set) overlap = matched / len(expected_tokens) if overlap >= 0.8: return 0.6 if overlap >= 0.5: return 0.3 return 0.0 # --------------------------------------------------------------------------- # Internal helper — extract most code-like block from agent response # --------------------------------------------------------------------------- def _extract_code(response: str) -> str: """ Extract the most likely code block from a free-text response. Strategy (in order): 1. Pull content from ```...``` fenced blocks if present. 2. Pull lines that look like code (contain { } ; = ( ) arrows). 3. Fall back to the full response stripped. """ # 1. Fenced code block fenced = re.findall(r"```(?:\w+)?\n?(.*?)```", response, re.DOTALL) if fenced: return max(fenced, key=len).strip() # 2. Lines that look like code code_lines = [] for line in response.splitlines(): stripped = line.strip() if re.search(r"[{}();=>\[\]]", stripped): code_lines.append(stripped) if code_lines: return "\n".join(code_lines) # 3. Full response return response.strip()