env/grader.py

"""
Deterministic graders for the Code Review environment.

Rules:
- No randomness, no LLM calls — fully deterministic.
- All functions return float in [0.0, 1.0].
- grade_identify: partial credit via keyword matching.
- grade_fix: strict match first, then token-normalized fallback.
"""

import re
from typing import List


# ---------------------------------------------------------------------------
# Identify grader — partial scoring
# ---------------------------------------------------------------------------

def grade_identify(expected_keywords: List[str], response: str) -> float:
    """
    Score an identify action by counting matched keywords.

    Scoring:
      - Smooth partial scoring: score = matched / total keywords.
      - Whole-word regex matching prevents substring false-positives.
      - No hard threshold — every matched keyword contributes proportionally.
      - Score is capped at 1.0.

    Args:
        expected_keywords: List of keywords the agent should mention.
        response: The agent's identify explanation (free text).

    Returns:
        float in [0.0, 1.0]
    """
    if not response or not expected_keywords:
        return 0.0

    normalized = response.lower()
    matched = [
        kw for kw in expected_keywords
        if re.search(r"\b" + re.escape(kw.lower()) + r"\b", normalized)
    ]

    score = len(matched) / len(expected_keywords)
    return round(min(score, 1.0), 2)


# ---------------------------------------------------------------------------
# Fix grader — strict + normalized fallback
# ---------------------------------------------------------------------------

def _normalize(code: str) -> str:
    """
    Normalize code for comparison:
    - Strip leading/trailing whitespace
    - Collapse all internal whitespace runs to a single space
    - Lowercase
    """
    code = code.strip()
    code = re.sub(r"\s+", " ", code)
    return code.lower()


def _tokenize(code: str) -> List[str]:
    """Split normalized code into alphanum tokens for partial overlap scoring."""
    return re.findall(r"[a-z0-9]+", _normalize(code))


def grade_fix(expected_code: str, response: str) -> float:
    """
    Score a fix action against the expected fixed code.

    Scoring tiers:
      1. Exact match (after stripping)          → 1.0
      2. Normalized match (whitespace collapsed) → 0.9
      3. Token overlap ≥ 80%                    → 0.6
      4. Token overlap ≥ 50%                    → 0.3
      5. Below 50% overlap                      → 0.0

    The response may contain surrounding explanation — we extract the
    largest code-like block for comparison before grading.

    Args:
        expected_code: The correct fixed version of the buggy code.
        response: The agent's fix response (may include explanation).

    Returns:
        float in [0.0, 1.0]
    """
    if not response:
        return 0.0

    # Tier 1 — exact strip match on raw response before any extraction
    if response.strip() == expected_code.strip():
        return 1.0

    candidate = _extract_code(response)

    # Tier 1b — exact strip match on extracted block
    if candidate.strip() == expected_code.strip():
        return 1.0

    # Tier 2 — normalized whitespace match
    if _normalize(candidate) == _normalize(expected_code):
        return 0.9

    # Tier 3 & 4 — token overlap
    expected_tokens = _tokenize(expected_code)
    candidate_tokens = _tokenize(candidate)

    if not expected_tokens:
        return 0.0

    # Overlap: count how many expected tokens appear in candidate
    candidate_set = set(candidate_tokens)
    matched = sum(1 for t in expected_tokens if t in candidate_set)
    overlap = matched / len(expected_tokens)

    if overlap >= 0.8:
        return 0.6
    if overlap >= 0.5:
        return 0.3

    return 0.0


# ---------------------------------------------------------------------------
# Internal helper — extract most code-like block from agent response
# ---------------------------------------------------------------------------

def _extract_code(response: str) -> str:
    """
    Extract the most likely code block from a free-text response.

    Strategy (in order):
      1. Pull content from ```...``` fenced blocks if present.
      2. Pull lines that look like code (contain { } ; = ( ) arrows).
      3. Fall back to the full response stripped.
    """
    # 1. Fenced code block
    fenced = re.findall(r"```(?:\w+)?\n?(.*?)```", response, re.DOTALL)
    if fenced:
        return max(fenced, key=len).strip()

    # 2. Lines that look like code
    code_lines = []
    for line in response.splitlines():
        stripped = line.strip()
        if re.search(r"[{}();=>\[\]]", stripped):
            code_lines.append(stripped)

    if code_lines:
        return "\n".join(code_lines)

    # 3. Full response
    return response.strip()