File size: 3,060 Bytes
c0e3412 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 | """Real HumanEval execution verifier using subprocess.
Replaces the proxy rubric-based verifier with actual test execution.
"""
from __future__ import annotations
import ast
import os
import subprocess
import sys
import tempfile
import textwrap
import traceback
from typing import Dict, List, Optional, Tuple
HUMANEVAL_TIMEOUT = 10 # seconds per test
def verify_humaneval(
generated_code: str,
test_code: str,
entry_point: str,
timeout: int = HUMANEVAL_TIMEOUT,
) -> Dict:
"""Execute the generated code against HumanEval test cases.
Args:
generated_code: The LLM-generated solution.
test_code: The HumanEval test assertions.
entry_point: Function name to test.
timeout: Max seconds per execution.
Returns:
Dict with:
- passed: bool
- execution_error: Optional error message
- stdout: captured stdout
- timing: execution time in seconds
"""
import time
# Build complete test script
test_script = textwrap.dedent(f"""
import sys
import traceback
# Generated solution
{generated_code}
# Test cases
{test_code}
# Run test
if __name__ == "__main__":
try:
check({entry_point})
print("PASSED")
except AssertionError as e:
print(f"FAILED: {{e}}")
except Exception as e:
print(f"ERROR: {{e}}")
traceback.print_exc()
""")
t0 = time.perf_counter()
try:
with tempfile.TemporaryDirectory() as tmpdir:
script_path = os.path.join(tmpdir, "test_script.py")
with open(script_path, "w") as f:
f.write(test_script)
result = subprocess.run(
[sys.executable, "-I", "-u", script_path],
capture_output=True,
text=True,
timeout=timeout,
cwd=tmpdir,
env={"PATH": "/usr/bin:/bin", "PYTHONPATH": ""},
)
dt = time.perf_counter() - t0
stdout = result.stdout.strip()
stderr = result.stderr.strip()
passed = "PASSED" in stdout
error = stderr if stderr else None
if "FAILED" in stdout:
error = stdout
return {
"passed": passed,
"execution_error": error,
"stdout": stdout[:500],
"timing": round(dt, 3),
}
except subprocess.TimeoutExpired:
return {
"passed": False,
"execution_error": f"Timeout after {timeout}s",
"stdout": "",
"timing": timeout,
}
except Exception as e:
return {
"passed": False,
"execution_error": str(e),
"stdout": "",
"timing": round(time.perf_counter() - t0, 3),
}
def is_valid_python(code: str) -> Tuple[bool, Optional[str]]:
"""Check if code is syntactically valid Python."""
try:
ast.parse(code)
return True, None
except SyntaxError as e:
return False, str(e)
|