agents/critic.py

"""
agents/critic.py - Anti-hacking checks for CodeDebugger.
"""

import re


# Common language constants that are legitimate return values, not cheating
EXCLUDED_CONSTANTS = {
    'True', 'False', 'None', '0', '1', '-1',
    '[]', '{}', '', 'true', 'false', 'null',
}


def check_hardcoded_outputs(code: str, test_cases: list) -> dict:
    """
    Detect if model hardcodes expected outputs.
    Check if any expected value from test_cases appears as a 
    literal return value in the code (e.g. "return 3" when 
    expected is "3").
    
    Excludes common language constants (True, False, None, 0, 1, etc.)
    and requires at least 2 matches before triggering penalty.
    
    Returns: {"passed": bool, "penalty": float, "message": str}
    Penalty if caught: -30
    """
    matches = []
    
    for tc in test_cases:
        expected = str(tc.get("expected", "")).strip()
        if not expected:
            continue
        
        # Skip common language constants — these are legitimate return values
        if expected in EXCLUDED_CONSTANTS:
            continue
            
        # Escape for regex to avoid syntax issues
        escaped_exp = re.escape(expected)
        
        # Match 'return <expected>', 'return "<expected>"', or 'return \'<expected>\''
        pattern = rf"return\s+({escaped_exp}|'{escaped_exp}'|\"{escaped_exp}\")"
        if re.search(pattern, code):
            matches.append(expected)
    
    # Require at least 2 hardcoded matches to trigger — a single coincidence is not cheating
    if len(matches) >= 2:
        return {"passed": False, "penalty": -30.0, "message": f"Hardcoded returns detected for values: {', '.join(matches)}"}
            
    return {"passed": True, "penalty": 0.0, "message": "OK"}


def check_function_deletion(original_code: str, submitted_code: str) -> dict:
    """
    Detect if function signature was removed or changed.
    Extract function name from original using regex.
    Check if same function name exists in submitted code.
    Returns: {"passed": bool, "penalty": float, "message": str}
    Penalty if caught: -25
    """
    match = re.search(r"def\s+([a-zA-Z0-9_]+)\s*\(", original_code)
    if match:
        func_name = match.group(1)
        if not re.search(rf"def\s+{func_name}\s*\(", submitted_code):
            return {"passed": False, "penalty": -25.0, "message": f"Function '{func_name}' was removed or renamed."}
            
    return {"passed": True, "penalty": 0.0, "message": "OK"}


def check_trivial_pass(test_results: dict, execution_time: float) -> dict:
    """
    Detect suspicious: all tests pass but execution < 0.01s.
    If pass_rate == 1.0 AND execution_time_total < 0.01: suspicious.
    Returns: {"passed": bool, "penalty": float, "message": str}
    Penalty if caught: -20
    """
    pass_rate = test_results.get("pass_rate", 0.0)
    exec_time = test_results.get("execution_time_total", execution_time)
    
    if pass_rate == 1.0 and exec_time < 0.01:
        return {"passed": False, "penalty": -20.0, "message": "Suspiciously fast execution for all passing tests."}
        
    return {"passed": True, "penalty": 0.0, "message": "OK"}


def check_code_gutting(original_code: str, submitted_code: str) -> dict:
    """
    Detect if model deleted most of the code.
    If len(submitted) < 0.3 * len(original): gutted.
    If submitted is just "pass" or "return None": gutted.
    Returns: {"passed": bool, "penalty": float, "message": str}
    Penalty if caught: -35
    """
    sub_strip = submitted_code.strip()
    # Check for empty or trivial single-line bodies
    if sub_strip in ["pass", "return None", "", "return"]:
        return {"passed": False, "penalty": -35.0, "message": "Code is trivially empty or just 'pass'/'return None'."}
        
    if len(submitted_code) < 0.3 * len(original_code):
        return {"passed": False, "penalty": -35.0, "message": "Code length is < 30% of original."}
        
    return {"passed": True, "penalty": 0.0, "message": "OK"}


def run_all_anti_hack_checks(code, original_code, test_cases, test_results, execution_time) -> dict:
    """
    Run all 4 checks.
    Returns: {
      "all_passed": bool,
      "total_penalty": float,
      "checks": {
        "hardcoded": result,
        "deletion": result,
        "trivial": result,
        "gutting": result
      }
    }
    """
    r_hardcoded = check_hardcoded_outputs(code, test_cases)
    r_deletion = check_function_deletion(original_code, code)
    r_trivial = check_trivial_pass(test_results, execution_time)
    r_gutting = check_code_gutting(original_code, code)
    
    checks = {
        "hardcoded": r_hardcoded,
        "deletion": r_deletion,
        "trivial": r_trivial,
        "gutting": r_gutting
    }
    
    all_passed = all(c["passed"] for c in checks.values())
    total_penalty = sum(c["penalty"] for c in checks.values())
    
    return {
        "all_passed": all_passed,
        "total_penalty": float(total_penalty),
        "checks": checks
    }