| """Real HumanEval execution verifier using subprocess. |
| Replaces the proxy rubric-based verifier with actual test execution. |
| """ |
| from __future__ import annotations |
| import ast |
| import os |
| import subprocess |
| import sys |
| import tempfile |
| import textwrap |
| import traceback |
| from typing import Dict, List, Optional, Tuple |
|
|
| HUMANEVAL_TIMEOUT = 10 |
|
|
| def verify_humaneval( |
| generated_code: str, |
| test_code: str, |
| entry_point: str, |
| timeout: int = HUMANEVAL_TIMEOUT, |
| ) -> Dict: |
| """Execute the generated code against HumanEval test cases. |
| |
| Args: |
| generated_code: The LLM-generated solution. |
| test_code: The HumanEval test assertions. |
| entry_point: Function name to test. |
| timeout: Max seconds per execution. |
| |
| Returns: |
| Dict with: |
| - passed: bool |
| - execution_error: Optional error message |
| - stdout: captured stdout |
| - timing: execution time in seconds |
| """ |
| import time |
|
|
| |
| test_script = textwrap.dedent(f""" |
| import sys |
| import traceback |
| |
| # Generated solution |
| {generated_code} |
| |
| # Test cases |
| {test_code} |
| |
| # Run test |
| if __name__ == "__main__": |
| try: |
| check({entry_point}) |
| print("PASSED") |
| except AssertionError as e: |
| print(f"FAILED: {{e}}") |
| except Exception as e: |
| print(f"ERROR: {{e}}") |
| traceback.print_exc() |
| """) |
|
|
| t0 = time.perf_counter() |
| try: |
| with tempfile.TemporaryDirectory() as tmpdir: |
| script_path = os.path.join(tmpdir, "test_script.py") |
| with open(script_path, "w") as f: |
| f.write(test_script) |
|
|
| result = subprocess.run( |
| [sys.executable, "-I", "-u", script_path], |
| capture_output=True, |
| text=True, |
| timeout=timeout, |
| cwd=tmpdir, |
| env={"PATH": "/usr/bin:/bin", "PYTHONPATH": ""}, |
| ) |
|
|
| dt = time.perf_counter() - t0 |
| stdout = result.stdout.strip() |
| stderr = result.stderr.strip() |
|
|
| passed = "PASSED" in stdout |
| error = stderr if stderr else None |
|
|
| if "FAILED" in stdout: |
| error = stdout |
|
|
| return { |
| "passed": passed, |
| "execution_error": error, |
| "stdout": stdout[:500], |
| "timing": round(dt, 3), |
| } |
| except subprocess.TimeoutExpired: |
| return { |
| "passed": False, |
| "execution_error": f"Timeout after {timeout}s", |
| "stdout": "", |
| "timing": timeout, |
| } |
| except Exception as e: |
| return { |
| "passed": False, |
| "execution_error": str(e), |
| "stdout": "", |
| "timing": round(time.perf_counter() - t0, 3), |
| } |
|
|
|
|
| def is_valid_python(code: str) -> Tuple[bool, Optional[str]]: |
| """Check if code is syntactically valid Python.""" |
| try: |
| ast.parse(code) |
| return True, None |
| except SyntaxError as e: |
| return False, str(e) |
|
|
