Spaces:
Running
Running
File size: 2,837 Bytes
72bc633 91f7972 72bc633 91f7972 72bc633 91f7972 72bc633 91f7972 72bc633 91f7972 72bc633 91f7972 72bc633 91f7972 72bc633 91f7972 72bc633 91f7972 72bc633 91f7972 72bc633 91f7972 72bc633 91f7972 72bc633 91f7972 72bc633 91f7972 72bc633 91f7972 72bc633 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 | """Tests for the 3-stage patch validation pipeline."""
from patchhawk.agent.sandbox import validate_patch
def test_syntax_error_detected():
"""Stage 1: py_compile catches a missing closing quote."""
scenario = {
"type": "functional",
"code_snippet": "print('hello')",
"unit_test_code": None,
"attack_type": None,
}
patch = "print('hello" # syntax error
success, msg, details = validate_patch(scenario, patch, use_docker=False)
assert success is False
assert msg == "Syntax error"
assert "error" in details
def test_unit_test_pass():
"""Stage 2: a correct patch passes the unit test."""
scenario = {
"type": "functional",
"code_snippet": "def add(a, b): return a + b",
"unit_test_code": "import code\nassert code.add(1, 2) == 3\n",
"attack_type": None,
}
patch = "def add(a, b):\n return a + b\n"
success, msg, details = validate_patch(scenario, patch, use_docker=False)
assert success is True
assert msg == "Patch is valid"
def test_unit_test_fail():
"""Stage 2: a broken patch fails the unit test."""
scenario = {
"type": "functional",
"code_snippet": "def add(a, b): return a + b",
"unit_test_code": "import code\nassert code.add(1, 2) == 3\n",
"attack_type": None,
}
patch = "def add(a, b):\n return a - b\n"
success, msg, details = validate_patch(scenario, patch, use_docker=False)
assert success is False
assert msg == "Unit test failed"
assert "test_output" in details
def test_vulnerability_remains():
"""Stage 3: re-attack detects that the backdoor was not removed."""
scenario = {
"type": "true_positive",
"attack_type": "obfuscated_exec",
"code_snippet": "print('Hello from hidden exec!')\ndef add(a, b): return a + b",
"unit_test_code": "import code\nassert code.add(1, 2) == 3\n",
}
# Patch keeps the backdoor print
patch = "print('Hello from hidden exec!')\ndef add(a, b):\n return a + b\n"
success, msg, details = validate_patch(scenario, patch, use_docker=False)
assert success is False
assert msg == "Vulnerability still accessible"
assert "evidence" in details
def test_clean_patch_passes_reattack():
"""Stage 3: a properly cleaned patch passes re-attack."""
scenario = {
"type": "true_positive",
"attack_type": "obfuscated_exec",
"code_snippet": "print('Hello from hidden exec!')\ndef add(a, b): return a + b",
"unit_test_code": "import code\nassert code.add(1, 2) == 3\n",
}
# Patch removes the backdoor
patch = "def add(a, b):\n return a + b\n"
success, msg, details = validate_patch(scenario, patch, use_docker=False)
assert success is True
assert msg == "Patch is valid"
|