environment/graders.py

from typing import List
from environment.models import Issue

def compute_f1(agent_issues: List[Issue], ground_truth: List[Issue]) -> float:
    """
    Deterministic grader: exact match on line and category.
    Returns F1 score between 0.0 and 1.0.
    """
    # Convert ground truth to set of (line, category) tuples
    truth_set = {(issue.line, issue.category) for issue in ground_truth}
    agent_set = {(issue.line, issue.category) for issue in agent_issues}

    true_positives = len(truth_set & agent_set)
    false_positives = len(agent_set - truth_set)
    false_negatives = len(truth_set - agent_set)

    precision = true_positives / (true_positives + false_positives) if (true_positives + false_positives) > 0 else 0.0
    recall = true_positives / (true_positives + false_negatives) if (true_positives + false_negatives) > 0 else 0.0

    if precision + recall == 0:
        return 0.0
    f1 = 2 * (precision * recall) / (precision + recall)
    return round(f1, 3)

def grade_easy(agent_issues: List[Issue]) -> float:
    from environment.tasks import TASKS
    return compute_f1(agent_issues, TASKS["easy"]["ground_truth"])

def grade_medium(agent_issues: List[Issue]) -> float:
    from environment.tasks import TASKS
    return compute_f1(agent_issues, TASKS["medium"]["ground_truth"])

def grade_hard(agent_issues: List[Issue]) -> float:
    from environment.tasks import TASKS
    return compute_f1(agent_issues, TASKS["hard"]["ground_truth"])