turnabout-bench / tests /test_validator.py
王致渊
Initial commit
57e0211
Raw
History Blame Contribute Delete
30.1 kB
"""Tests for the case data validator."""
import json
from pathlib import Path
import pytest
from turnabout.core.schema import CaseData
from turnabout.generation.validator import CaseValidator, ValidationReport
CASES_DIR = Path(__file__).parent.parent / "turnabout" / "cases"
# ------------------------------------------------------------------
# Fixtures
# ------------------------------------------------------------------
@pytest.fixture
def validator():
return CaseValidator()
@pytest.fixture
def stolen_prototype():
path = CASES_DIR / "stolen_prototype.json"
if not path.exists():
pytest.skip("stolen_prototype.json not found")
with open(path) as f:
data = json.load(f)
return CaseData(**data)
def _make_minimal_case(**overrides) -> dict:
"""Build a minimal valid case dict for mutation-based tests."""
base = {
"id": "test_case",
"title": "Test Case",
"difficulty": "easy",
"description": "A test case.",
"characters": [
{"id": "defendant", "name": "D", "description": "d", "role": "defendant"},
{"id": "witness1", "name": "W", "description": "w", "role": "witness"},
],
"evidence": [
{
"id": "ev1",
"name": "E1",
"item_type": "document",
"description": "desc",
"detail": "detail",
"source": "pre_given",
},
{
"id": "ev2",
"name": "E2",
"item_type": "document",
"description": "desc",
"detail": "detail",
"source": "pre_given",
},
],
"court": {
"penalty_limit": 5,
"rounds": [
{
"witness_id": "witness1",
"order": 0,
"initial_testimony_id": "t1",
"testimonies": [
{
"id": "t1",
"title": "Testimony",
"witness_id": "witness1",
"preamble": "The witness testifies.",
"statements": [
{
"id": "s1",
"text": "I saw something.",
"press_response": "Well...",
"contradiction": {
"evidence_id": "ev1",
"explanation": "This contradicts!",
"is_primary": True,
},
},
{
"id": "s2",
"text": "I was there.",
"press_response": "Yes...",
"contradiction": None,
},
],
}
],
}
],
"win_condition": {"required_contradiction_ids": ["s1"]},
},
}
base.update(overrides)
return base
def _make_minimal_case_data(**overrides) -> CaseData:
return CaseData(**_make_minimal_case(**overrides))
# ------------------------------------------------------------------
# Tests: ValidationReport
# ------------------------------------------------------------------
class TestValidationReport:
def test_default_is_valid(self):
report = ValidationReport()
assert report.is_valid is True
assert report.errors == []
assert report.warnings == []
def test_add_error_makes_invalid(self):
report = ValidationReport()
report.add_error("something broke")
assert report.is_valid is False
assert len(report.errors) == 1
def test_add_warning_stays_valid(self):
report = ValidationReport()
report.add_warning("minor issue")
assert report.is_valid is True
assert len(report.warnings) == 1
# ------------------------------------------------------------------
# Tests: Valid cases
# ------------------------------------------------------------------
class TestValidCases:
def test_stolen_prototype_is_valid(self, validator, stolen_prototype):
report = validator.validate(stolen_prototype)
assert report.is_valid, f"Errors: {report.errors}"
def test_minimal_case_is_valid(self, validator):
case = _make_minimal_case_data()
report = validator.validate(case)
assert report.is_valid, f"Errors: {report.errors}"
# ------------------------------------------------------------------
# Tests: Reference validation
# ------------------------------------------------------------------
class TestReferenceValidation:
def test_bad_location_connection(self, validator):
data = _make_minimal_case()
data["locations"] = [
{
"id": "loc1",
"name": "L1",
"description": "d",
"connected_to": ["nonexistent_loc"],
"available_from_start": True,
}
]
case = CaseData(**data)
report = validator.validate(case)
assert not report.is_valid
assert any("nonexistent_loc" in e for e in report.errors)
def test_bad_character_in_location(self, validator):
data = _make_minimal_case()
data["locations"] = [
{
"id": "loc1",
"name": "L1",
"description": "d",
"characters_present": ["ghost_character"],
"available_from_start": True,
}
]
case = CaseData(**data)
report = validator.validate(case)
assert not report.is_valid
assert any("ghost_character" in e for e in report.errors)
def test_bad_evidence_acquisition_location(self, validator):
data = _make_minimal_case()
data["locations"] = [
{
"id": "loc1",
"name": "L1",
"description": "d",
"examinables": [
{"id": "obj1", "name": "O1", "description": "d"}
],
"available_from_start": True,
}
]
data["evidence"].append(
{
"id": "ev3",
"name": "E3",
"item_type": "physical",
"description": "d",
"detail": "d",
"source": "investigation",
"acquisition": {
"method": "examine",
"location_id": "bad_loc",
"target_id": "obj1",
},
}
)
case = CaseData(**data)
report = validator.validate(case)
assert not report.is_valid
assert any("bad_loc" in e for e in report.errors)
def test_bad_evidence_acquisition_target(self, validator):
data = _make_minimal_case()
data["locations"] = [
{
"id": "loc1",
"name": "L1",
"description": "d",
"examinables": [],
"available_from_start": True,
}
]
data["evidence"].append(
{
"id": "ev3",
"name": "E3",
"item_type": "physical",
"description": "d",
"detail": "d",
"source": "investigation",
"acquisition": {
"method": "examine",
"location_id": "loc1",
"target_id": "bad_target",
},
}
)
case = CaseData(**data)
report = validator.validate(case)
assert not report.is_valid
assert any("bad_target" in e for e in report.errors)
def test_bad_win_condition_statement(self, validator):
data = _make_minimal_case()
data["court"]["win_condition"]["required_contradiction_ids"] = [
"nonexistent_statement"
]
case = CaseData(**data)
report = validator.validate(case)
assert not report.is_valid
assert any("nonexistent_statement" in e for e in report.errors)
def test_bad_dialogue_character(self, validator):
data = _make_minimal_case()
data["dialogues"] = [
{
"character_id": "nobody",
"lines": [],
"present_responses": [],
}
]
case = CaseData(**data)
report = validator.validate(case)
assert not report.is_valid
assert any("nobody" in e for e in report.errors)
def test_bad_court_round_witness(self, validator):
data = _make_minimal_case()
data["court"]["rounds"][0]["witness_id"] = "missing_witness"
case = CaseData(**data)
report = validator.validate(case)
assert not report.is_valid
assert any("missing_witness" in e for e in report.errors)
def test_bad_triggers_testimony(self, validator):
data = _make_minimal_case()
stmt = data["court"]["rounds"][0]["testimonies"][0]["statements"][0]
stmt["contradiction"]["triggers_testimony"] = "phantom_testimony"
case = CaseData(**data)
report = validator.validate(case)
assert not report.is_valid
assert any("phantom_testimony" in e for e in report.errors)
def test_bad_press_reveals_statement(self, validator):
data = _make_minimal_case()
stmt = data["court"]["rounds"][0]["testimonies"][0]["statements"][0]
stmt["press_reveals_statement"] = "ghost_statement"
case = CaseData(**data)
report = validator.validate(case)
assert not report.is_valid
assert any("ghost_statement" in e for e in report.errors)
def test_bad_final_evidence_id(self, validator):
data = _make_minimal_case()
data["court"]["win_condition"]["final_evidence_id"] = "missing_ev"
case = CaseData(**data)
report = validator.validate(case)
assert not report.is_valid
assert any("missing_ev" in e for e in report.errors)
def test_bad_investigation_gate_evidence(self, validator):
data = _make_minimal_case()
data["investigation_gate"] = {
"required_evidence": ["nonexistent_ev"],
"auto_advance": False,
}
case = CaseData(**data)
report = validator.validate(case)
assert not report.is_valid
assert any("nonexistent_ev" in e for e in report.errors)
def test_bad_location_unlock_prerequisite(self, validator):
data = _make_minimal_case()
data["locations"] = [
{
"id": "loc1",
"name": "L1",
"description": "d",
"available_from_start": True,
"connected_to": ["loc2"],
},
{
"id": "loc2",
"name": "L2",
"description": "d",
"available_from_start": False,
"unlock_prerequisites": ["fake_ev"],
"connected_to": ["loc1"],
},
]
case = CaseData(**data)
report = validator.validate(case)
assert not report.is_valid
assert any("fake_ev" in e for e in report.errors)
def test_bad_dialogue_line_grants_evidence(self, validator):
data = _make_minimal_case()
data["dialogues"] = [
{
"character_id": "witness1",
"lines": [
{
"id": "line1",
"text": "hi",
"grants_evidence": "missing_ev",
}
],
"present_responses": [],
}
]
case = CaseData(**data)
report = validator.validate(case)
assert not report.is_valid
assert any("missing_ev" in e for e in report.errors)
def test_bad_present_response_evidence(self, validator):
data = _make_minimal_case()
data["dialogues"] = [
{
"character_id": "witness1",
"lines": [],
"present_responses": [
{
"evidence_id": "ghost_ev",
"text": "hmm",
}
],
}
]
case = CaseData(**data)
report = validator.validate(case)
assert not report.is_valid
assert any("ghost_ev" in e for e in report.errors)
# ------------------------------------------------------------------
# Tests: Contradiction validation
# ------------------------------------------------------------------
class TestContradictionValidation:
def test_bad_contradiction_evidence(self, validator):
data = _make_minimal_case()
stmt = data["court"]["rounds"][0]["testimonies"][0]["statements"][0]
stmt["contradiction"]["evidence_id"] = "nonexistent_ev"
case = CaseData(**data)
report = validator.validate(case)
assert not report.is_valid
assert any("nonexistent_ev" in e for e in report.errors)
def test_valid_contradiction_evidence(self, validator):
case = _make_minimal_case_data()
errors = validator._validate_contradictions(case)
assert errors == []
# ------------------------------------------------------------------
# Tests: Solvability validation
# ------------------------------------------------------------------
class TestSolvabilityValidation:
def test_win_condition_statement_without_contradiction(self, validator):
data = _make_minimal_case()
# s2 has no contradiction but we require it in win condition
data["court"]["win_condition"]["required_contradiction_ids"] = ["s2"]
case = CaseData(**data)
report = validator.validate(case)
assert not report.is_valid
assert any("no contradiction defined" in e for e in report.errors)
def test_unreachable_testimony_for_win(self, validator):
data = _make_minimal_case()
# Add a second testimony that is never triggered
data["court"]["rounds"][0]["testimonies"].append(
{
"id": "t2",
"title": "Secret Testimony",
"witness_id": "witness1",
"preamble": "Hidden.",
"statements": [
{
"id": "s3",
"text": "Secret statement.",
"press_response": "...",
"contradiction": {
"evidence_id": "ev2",
"explanation": "Objection!",
"is_primary": True,
},
}
],
}
)
# Require s3 which is in unreachable t2
data["court"]["win_condition"]["required_contradiction_ids"] = ["s1", "s3"]
case = CaseData(**data)
report = validator.validate(case)
assert not report.is_valid
assert any("unreachable" in e.lower() for e in report.errors)
def test_reachable_testimony_via_trigger(self, validator):
"""Testimony reachable through triggers_testimony chain should pass."""
data = _make_minimal_case()
# s1 contradiction triggers t2
stmt = data["court"]["rounds"][0]["testimonies"][0]["statements"][0]
stmt["contradiction"]["triggers_testimony"] = "t2"
data["court"]["rounds"][0]["testimonies"].append(
{
"id": "t2",
"title": "Second Testimony",
"witness_id": "witness1",
"preamble": "More.",
"statements": [
{
"id": "s3",
"text": "Second statement.",
"press_response": "...",
"contradiction": {
"evidence_id": "ev2",
"explanation": "Objection!",
"is_primary": True,
},
}
],
}
)
data["court"]["win_condition"]["required_contradiction_ids"] = ["s1", "s3"]
case = CaseData(**data)
report = validator.validate(case)
assert report.is_valid, f"Errors: {report.errors}"
# ------------------------------------------------------------------
# Tests: Reachability validation
# ------------------------------------------------------------------
class TestReachabilityValidation:
def test_unreachable_investigation_evidence(self, validator):
data = _make_minimal_case()
data["locations"] = [
{
"id": "loc1",
"name": "L1",
"description": "d",
"examinables": [
{"id": "obj1", "name": "O1", "description": "d"}
],
"available_from_start": True,
"connected_to": ["loc2"],
},
{
"id": "loc2",
"name": "L2",
"description": "d",
"examinables": [
{"id": "obj2", "name": "O2", "description": "d"}
],
"available_from_start": False,
# Requires ev3 to unlock, but ev3 is IN loc2 -- catch-22
"unlock_prerequisites": ["ev3"],
"connected_to": ["loc1"],
},
]
data["evidence"].append(
{
"id": "ev3",
"name": "E3",
"item_type": "physical",
"description": "d",
"detail": "d",
"source": "investigation",
"acquisition": {
"method": "examine",
"location_id": "loc2",
"target_id": "obj2",
},
}
)
case = CaseData(**data)
report = validator.validate(case)
assert not report.is_valid
assert any("unreachable" in e.lower() for e in report.errors)
def test_reachable_evidence_chain(self, validator):
"""Evidence obtainable through a prerequisite chain should pass."""
data = _make_minimal_case()
data["locations"] = [
{
"id": "loc1",
"name": "L1",
"description": "d",
"examinables": [
{"id": "obj1", "name": "O1", "description": "d"},
{"id": "obj2", "name": "O2", "description": "d"},
],
"available_from_start": True,
}
]
data["evidence"].extend(
[
{
"id": "ev3",
"name": "E3",
"item_type": "physical",
"description": "d",
"detail": "d",
"source": "investigation",
"acquisition": {
"method": "examine",
"location_id": "loc1",
"target_id": "obj1",
"prerequisites": [],
},
},
{
"id": "ev4",
"name": "E4",
"item_type": "physical",
"description": "d",
"detail": "d",
"source": "investigation",
"acquisition": {
"method": "examine",
"location_id": "loc1",
"target_id": "obj2",
"prerequisites": ["ev3"],
},
},
]
)
case = CaseData(**data)
report = validator.validate(case)
assert report.is_valid, f"Errors: {report.errors}"
def test_stolen_prototype_reachable(self, validator, stolen_prototype):
"""All evidence in stolen_prototype should be reachable."""
errors = validator._validate_reachability(stolen_prototype)
assert errors == [], f"Reachability errors: {errors}"
def test_unreachable_gate_evidence(self, validator):
"""Investigation gate requiring unobtainable evidence should fail."""
data = _make_minimal_case()
data["locations"] = [
{
"id": "loc1",
"name": "L1",
"description": "d",
"examinables": [],
"available_from_start": True,
}
]
# ev3 is investigation evidence but has no way to be obtained
# (its location does not contain the right examinable)
data["evidence"].append(
{
"id": "ev3",
"name": "E3",
"item_type": "physical",
"description": "d",
"detail": "d",
"source": "investigation",
"acquisition": {
"method": "examine",
"location_id": "loc1",
"target_id": "missing_obj",
},
}
)
data["investigation_gate"] = {
"required_evidence": ["ev3"],
"auto_advance": False,
}
case = CaseData(**data)
report = validator.validate(case)
assert not report.is_valid
assert any("unreachable" in e.lower() for e in report.errors)
def test_evidence_from_dialogue(self, validator):
"""Evidence obtainable via dialogue should be reachable."""
data = _make_minimal_case()
data["locations"] = [
{
"id": "loc1",
"name": "L1",
"description": "d",
"characters_present": ["witness1"],
"available_from_start": True,
}
]
data["evidence"].append(
{
"id": "ev3",
"name": "E3",
"item_type": "testimony_record",
"description": "d",
"detail": "d",
"source": "investigation",
"acquisition": {
"method": "talk",
"location_id": "loc1",
"target_id": "witness1",
},
}
)
data["dialogues"] = [
{
"character_id": "witness1",
"lines": [
{
"id": "line1",
"text": "Here is some info.",
"grants_evidence": "ev3",
}
],
"present_responses": [],
}
]
case = CaseData(**data)
report = validator.validate(case)
assert report.is_valid, f"Errors: {report.errors}"
# ------------------------------------------------------------------
# Tests: Deadlock detection
# ------------------------------------------------------------------
class TestDeadlockValidation:
def test_circular_evidence_prerequisites(self, validator):
data = _make_minimal_case()
data["locations"] = [
{
"id": "loc1",
"name": "L1",
"description": "d",
"examinables": [
{"id": "obj1", "name": "O1", "description": "d"},
{"id": "obj2", "name": "O2", "description": "d"},
],
"available_from_start": True,
}
]
# ev3 requires ev4, ev4 requires ev3 -- circular
data["evidence"].extend(
[
{
"id": "ev3",
"name": "E3",
"item_type": "physical",
"description": "d",
"detail": "d",
"source": "investigation",
"acquisition": {
"method": "examine",
"location_id": "loc1",
"target_id": "obj1",
"prerequisites": ["ev4"],
},
},
{
"id": "ev4",
"name": "E4",
"item_type": "physical",
"description": "d",
"detail": "d",
"source": "investigation",
"acquisition": {
"method": "examine",
"location_id": "loc1",
"target_id": "obj2",
"prerequisites": ["ev3"],
},
},
]
)
case = CaseData(**data)
report = validator.validate(case)
assert not report.is_valid
assert any("circular" in e.lower() for e in report.errors)
def test_disconnected_location(self, validator):
data = _make_minimal_case()
data["locations"] = [
{
"id": "loc1",
"name": "L1",
"description": "d",
"connected_to": [],
"available_from_start": True,
},
{
"id": "loc2",
"name": "L2",
"description": "d",
"connected_to": [],
"available_from_start": False,
"unlock_prerequisites": [],
},
]
case = CaseData(**data)
report = validator.validate(case)
assert not report.is_valid
assert any("not reachable" in e.lower() for e in report.errors)
def test_no_deadlocks_in_stolen_prototype(self, validator, stolen_prototype):
errors = validator._validate_no_deadlocks(stolen_prototype)
assert errors == [], f"Deadlock errors: {errors}"
# ------------------------------------------------------------------
# Tests: Structural warnings
# ------------------------------------------------------------------
class TestStructuralWarnings:
def test_no_defendant_warning(self, validator):
data = _make_minimal_case()
data["characters"] = [
{"id": "npc1", "name": "N", "description": "n", "role": "npc"},
{"id": "witness1", "name": "W", "description": "w", "role": "witness"},
]
case = CaseData(**data)
report = validator.validate(case)
assert any("defendant" in w.lower() for w in report.warnings)
def test_hard_mode_no_locations_warning(self, validator):
data = _make_minimal_case(difficulty="hard")
case = CaseData(**data)
report = validator.validate(case)
assert any("locations" in w.lower() for w in report.warnings)
def test_hard_mode_no_dialogues_warning(self, validator):
data = _make_minimal_case(difficulty="hard")
data["locations"] = [
{
"id": "loc1",
"name": "L1",
"description": "d",
"available_from_start": True,
}
]
case = CaseData(**data)
report = validator.validate(case)
assert any("dialogues" in w.lower() for w in report.warnings)
def test_hard_mode_no_gate_warning(self, validator):
data = _make_minimal_case(difficulty="hard")
data["locations"] = [
{
"id": "loc1",
"name": "L1",
"description": "d",
"available_from_start": True,
}
]
data["dialogues"] = []
case = CaseData(**data)
report = validator.validate(case)
assert any("investigation gate" in w.lower() for w in report.warnings)
def test_isolated_location_warning(self, validator):
data = _make_minimal_case()
data["locations"] = [
{
"id": "loc1",
"name": "L1",
"description": "d",
"connected_to": [],
"available_from_start": True,
},
{
"id": "loc2",
"name": "L2",
"description": "d",
"connected_to": [],
"available_from_start": True,
},
]
case = CaseData(**data)
report = validator.validate(case)
assert any("no connections" in w.lower() for w in report.warnings)
# ------------------------------------------------------------------
# Tests: Full validate() integration
# ------------------------------------------------------------------
class TestFullValidation:
def test_stolen_prototype_full_validation(self, validator, stolen_prototype):
report = validator.validate(stolen_prototype)
assert report.is_valid, f"Errors: {report.errors}"
# Stolen prototype is a well-formed hard case; expect no warnings
# (or at most minor ones)
def test_multiple_errors_reported(self, validator):
"""Multiple issues should all be reported."""
data = _make_minimal_case()
data["court"]["win_condition"]["required_contradiction_ids"] = [
"bad_stmt_1",
"bad_stmt_2",
]
case = CaseData(**data)
report = validator.validate(case)
assert not report.is_valid
assert len(report.errors) >= 2
def test_clean_minimal_case(self, validator):
case = _make_minimal_case_data()
report = validator.validate(case)
assert report.is_valid
assert report.errors == []