"""Regression tests for the citation-grounded extraction layer. These tests are the executable success criteria for the anti-hallucination guarantees. They are fully deterministic and require NO OpenAI key — they exercise the pure-Python validation/contradiction core that gates the LLM. Covered failure modes (from the spec's TESTING REQUIREMENTS): - altered condition ratings - fabricated materials - invented locations / entity substitution - unsupported risk / severity claims - contradiction generation (rating / condition / operational / duplicate) - malformed / fabricated sentences (paraphrase with no grounding) - dropped warranties (positive findings preserved when grounded) - fabricated monetary totals - citation to a non-existent chunk """ from __future__ import annotations from app.extraction.citation_validator import ( span_in_chunk, validate_finding, validate_findings, ) from app.extraction.contradiction import audit_contradictions from app.extraction.schemas import ( ConditionRating, ContradictionKind, EvidenceSpan, SupportLevel, SurveyFinding, ) def _finding(element: str, rating, text: str, *, chunk_id="c1", span=None) -> SurveyFinding: return SurveyFinding( section="Roofing", element=element, condition_rating=rating, finding=text, evidence=[EvidenceSpan(chunk_id=chunk_id, text=span or text)], ) # ── ConditionRating enum: closed set, never fabricated ────────────────────── def test_rating_enum_coerces_unknown_to_na_not_a_guess(): assert ConditionRating.coerce("2") is ConditionRating.CR2 assert ConditionRating.coerce("CR3") is ConditionRating.CR3 # Garbage must degrade to NA, never to a fabricated severity. assert ConditionRating.coerce("urgent") is ConditionRating.NA assert ConditionRating.coerce("") is ConditionRating.NA assert ConditionRating.coerce(None) is ConditionRating.NA def test_finding_rating_is_schema_constrained(): f = SurveyFinding(section="Roofing", element="Ridge", condition_rating="severe", finding="x") assert f.condition_rating is ConditionRating.NA # ── span_in_chunk: verbatim + OCR-drift tolerance, but rejects absent text ── def test_span_match_verbatim_and_drift(): chunk = "The main roof covering is natural slate, generally in sound condition." assert span_in_chunk("natural slate", chunk) assert span_in_chunk("The main roof covering is natural slate", chunk) # whitespace/case drift still matches assert span_in_chunk("NATURAL slate", chunk) def test_span_absent_is_rejected(): chunk = "The main roof covering is natural slate." assert not span_in_chunk("concrete interlocking tiles", chunk) # ── Altered condition ratings are caught by the contradiction audit ───────── def test_altered_rating_conflict_detected_and_resolved(): strong = _finding("Main roof", ConditionRating.CR2, "Slate covering shows slipped tiles.") strong.support = SupportLevel.SUPPORTED weak = _finding("Main roof", ConditionRating.CR1, "Slate covering shows slipped tiles.") weak.support = SupportLevel.PARTIAL resolved, reports = audit_contradictions([strong, weak]) assert len(resolved) == 1 assert resolved[0].condition_rating is ConditionRating.CR2 # stronger evidence kept assert any(r.kind is ContradictionKind.RATING_CONFLICT for r in reports) # ── Fabricated materials / invented locations: entity substitution ────────── def test_entity_substitution_is_dropped(): pool = {"c1": "A London plane tree is located near the rear boundary."} # Model swapped the species — must be rejected as unsupported entity. bad = SurveyFinding( section="Grounds", element="Tree", condition_rating="NA", finding="A Lombardy Poplar is located near the rear boundary.", evidence=[EvidenceSpan(chunk_id="c1", text="located near the rear boundary")], ) support, violations = validate_finding(bad, pool) assert support is SupportLevel.NOT_FOUND assert any("entity" in v for v in violations) def test_correct_entity_is_supported(): pool = {"c1": "A London plane tree is located near the rear boundary."} good = SurveyFinding( section="Grounds", element="Tree", condition_rating="NA", finding="A London plane tree is located near the rear boundary.", evidence=[EvidenceSpan(chunk_id="c1", text="A London plane tree is located near the rear boundary")], ) support, violations = validate_finding(good, pool) assert support is SupportLevel.SUPPORTED assert violations == [] # ── Fabricated monetary totals / numbers ──────────────────────────────────── def test_fabricated_total_is_dropped(): pool = {"c1": "Repairs to the parapet are recommended."} bad = SurveyFinding( section="Roofing", element="Parapet", condition_rating="2", finding="Repairs to the parapet are recommended at a cost of £12,500.", evidence=[EvidenceSpan(chunk_id="c1", text="Repairs to the parapet are recommended")], ) support, violations = validate_finding(bad, pool) assert support is SupportLevel.NOT_FOUND assert any("number" in v or "amount" in v for v in violations) def test_grounded_total_is_preserved(): pool = {"c1": "Repairs to the parapet are recommended at a cost of £12,500."} good = SurveyFinding( section="Roofing", element="Parapet", condition_rating="2", finding="Repairs to the parapet are recommended at a cost of £12,500.", evidence=[EvidenceSpan(chunk_id="c1", text="Repairs to the parapet are recommended at a cost of £12,500")], ) support, _ = validate_finding(good, pool) assert support is SupportLevel.SUPPORTED # ── Unsupported risk / severity escalation ────────────────────────────────── def test_unsupported_severity_is_dropped(): pool = {"c1": "There is minor surface staining to the ceiling."} bad = SurveyFinding( section="Interior", element="Ceiling", condition_rating="2", finding="There is minor surface staining to the ceiling, a catastrophic and unsafe defect.", evidence=[EvidenceSpan(chunk_id="c1", text="There is minor surface staining to the ceiling")], ) support, violations = validate_finding(bad, pool) assert support is SupportLevel.NOT_FOUND assert any("severity" in v for v in violations) def test_severity_allowed_when_in_source(): pool = {"c1": "The boiler flue is unsafe and must not be used."} good = SurveyFinding( section="Services", element="Boiler flue", condition_rating="3", finding="The boiler flue is unsafe and must not be used.", evidence=[EvidenceSpan(chunk_id="c1", text="The boiler flue is unsafe and must not be used")], ) support, _ = validate_finding(good, pool) assert support is SupportLevel.SUPPORTED # ── Citation to a non-existent chunk ──────────────────────────────────────── def test_citation_to_unknown_chunk_is_rejected(): pool = {"c1": "Slate covering is sound."} bad = SurveyFinding( section="Roofing", element="Covering", condition_rating="1", finding="Slate covering is sound.", evidence=[EvidenceSpan(chunk_id="ghost", text="Slate covering is sound")], ) support, violations = validate_finding(bad, pool) assert support is SupportLevel.NOT_FOUND assert any("unknown chunk_id" in v for v in violations) # ── Malformed / paraphrased fabrication with no grounding ─────────────────── def test_ungrounded_paraphrase_is_dropped(): pool = {"c1": "The flat roof is covered in felt."} bad = SurveyFinding( section="Roofing", element="Flat roof", condition_rating="2", finding="Extensive structural movement threatens imminent failure of the dwelling.", evidence=[EvidenceSpan(chunk_id="c1", text="The flat roof is covered in felt")], ) support, _ = validate_finding(bad, pool) assert support is SupportLevel.NOT_FOUND # ── Positive findings / warranties preserved when grounded ────────────────── def test_warranty_preserved(): pool = {"c1": "The replacement boiler was fitted in 2021 and carries a 10 year manufacturer warranty."} good = SurveyFinding( section="Services", element="Boiler", condition_rating="1", finding="The replacement boiler was fitted in 2021 and carries a 10 year manufacturer warranty.", evidence=[EvidenceSpan( chunk_id="c1", text="The replacement boiler was fitted in 2021 and carries a 10 year manufacturer warranty", )], ) support, _ = validate_finding(good, pool) assert support is SupportLevel.SUPPORTED # ── Contradiction: satisfactory vs defective ──────────────────────────────── def test_satisfactory_vs_defective_contradiction(): a = _finding("Gutters", ConditionRating.NA, "The gutters are in good condition and sound.") a.support = SupportLevel.SUPPORTED b = _finding("Gutters", ConditionRating.NA, "The gutters are defective and leaking badly.") b.support = SupportLevel.NOT_FOUND resolved, reports = audit_contradictions([a, b]) assert len(resolved) == 1 assert resolved[0] is a # evidence-stronger side kept assert any(r.kind is ContradictionKind.CONDITION for r in reports) # ── Contradiction: operational vs non-operational ─────────────────────────── def test_operational_contradiction(): a = _finding("Heating", ConditionRating.NA, "The heating system is fully operational and working.") a.support = SupportLevel.SUPPORTED b = _finding("Heating", ConditionRating.NA, "The heating system is not operational and out of order.") b.support = SupportLevel.PARTIAL resolved, reports = audit_contradictions([a, b]) assert len(resolved) == 1 assert any(r.kind is ContradictionKind.OPERATIONAL for r in reports) # ── Duplicate collapse ────────────────────────────────────────────────────── def test_duplicate_findings_collapsed(): a = _finding("Chimney", ConditionRating.CR2, "The chimney stack shows perished pointing requiring repair.") b = _finding("Chimney", ConditionRating.CR2, "The chimney stack shows perished pointing requiring repair.") resolved, reports = audit_contradictions([a, b]) assert len(resolved) == 1 assert any(r.kind is ContradictionKind.DUPLICATE for r in reports) # ── End-to-end gate: mixed batch keeps only grounded, non-conflicting ─────── def test_validate_findings_batch_drops_unsupported(): pool = { "c1": "The main roof is natural slate in sound condition.", "c2": "The rear addition roof is covered in felt with no visible defects.", } findings = [ SurveyFinding(section="Roofing", element="Main roof", condition_rating="1", finding="The main roof is natural slate in sound condition.", evidence=[EvidenceSpan(chunk_id="c1", text="The main roof is natural slate in sound condition")]), SurveyFinding(section="Roofing", element="Rear roof", condition_rating="3", finding="The rear addition roof has collapsed and is deadly.", evidence=[EvidenceSpan(chunk_id="c2", text="The rear addition roof is covered in felt")]), ] kept, dropped = validate_findings(findings, pool) assert len(kept) == 1 assert kept[0].element == "Main roof" assert len(dropped) == 1 """Section-domain scoping (STEP 6) — prevent cross-section contamination.""" def test_classify_section_aliases_and_keywords(): from app.extraction.domain_scope import classify_section assert classify_section("Roof coverings") == "roofing" assert classify_section("Chimney stacks") == "chimney" assert classify_section("Rainwater pipes and gutters") == "rainwater" assert classify_section("Electricity") == "electrical" assert classify_section("About the property") == "general" def test_classify_text_dominant_domain(): from app.extraction.domain_scope import classify_text assert classify_text("The natural slate roof covering has slipped tiles at the ridge.") == "roofing" assert classify_text("The consumer unit lacks RCD protection on the circuits.") == "electrical" assert classify_text("This paragraph is generic boilerplate with no domain.") == "general" def test_scope_chunks_excludes_foreign_domain(): from app.extraction.domain_scope import scope_chunks class _Row: def __init__(self, text): self.text = text chunks = [ _Row("The slate roof covering is sound at the ridge and eaves."), _Row("The drainage manhole and inspection chamber were inspected."), _Row("General introductory text about the inspection."), ] kept = scope_chunks("roofing", chunks) texts = [c.text for c in kept] assert any("slate roof" in t for t in texts) assert any("introductory" in t for t in texts) # general is admissible assert not any("manhole" in t for t in texts) # drainage excluded def test_scope_chunks_never_starves_under_strict(): from app.extraction.domain_scope import scope_chunks class _Row: def __init__(self, text): self.text = text # All chunks belong to a different domain -> strict returns originals # rather than an empty pool. chunks = [_Row("The consumer unit and wiring circuits were inspected.")] kept = scope_chunks("roofing", chunks, strict=True) assert len(kept) == 1 kept_nonstrict = scope_chunks("roofing", chunks, strict=False) assert kept_nonstrict == [] def test_section_extraction_confidence(): pool = {"c1": "The main roof is natural slate in sound condition."} f = SurveyFinding(section="Roofing", element="Main roof", condition_rating="1", finding="The main roof is natural slate in sound condition.", evidence=[EvidenceSpan(chunk_id="c1", text="The main roof is natural slate in sound condition")]) kept, _ = validate_findings([f], pool) from app.extraction.schemas import SectionExtraction sec = SectionExtraction(section="Roofing", findings=kept) assert sec.confidence == 1.0 """Post-generation output validator + abstention (forbidden phrases / metrics).""" def test_output_validator_flags_forbidden_phrases(): from app.extraction.output_validator import find_violations, is_clean bad = "The roof appears to be defective and overall reliability is questionable." v = find_violations(bad, evidence="") assert any("appears to" in x for x in v) assert any("overall" in x for x in v) assert not is_clean(bad) def test_output_validator_flags_fabricated_percentage_and_score(): from app.extraction.output_validator import find_violations v = find_violations("Authenticity score is high with 87% confidence.", evidence="") assert any("percentage" in x for x in v) assert any("metric" in x for x in v) def test_output_validator_allows_terms_present_in_evidence(): from app.extraction.output_validator import is_clean # "unsafe" is permitted because it is verbatim in the evidence. text = "The flue is unsafe." evidence = "The boiler flue is unsafe and must not be used." assert is_clean(text, evidence) def test_output_validator_clean_text_passes(): from app.extraction.output_validator import is_clean assert is_clean("The main roof covering is natural slate.", evidence="") def test_abstain_on_low_confidence_and_attribution_failure(): from app.extraction.output_validator import should_abstain assert should_abstain("clean text", "", confidence=0.5, min_confidence=1.0) assert should_abstain("clean text", "", evidence_aligned=False) assert should_abstain("clean text", "", source_attributed=False) assert not should_abstain("The roof is slate.", "", confidence=1.0) def test_findings_to_atomic_claims_grounded(): from app.extraction.extractor import findings_to_atomic_claims from app.extraction.schemas import ClaimType f = SurveyFinding( section="Roofing", element="Main roof", condition_rating="2", finding="The main roof covering is natural slate with slipped tiles.", evidence=[EvidenceSpan(chunk_id="c1", page=12, section_label="Page 12", text="The main roof covering is natural slate with slipped tiles")], ) f.support = SupportLevel.SUPPORTED claims = findings_to_atomic_claims([f]) types = {c.claim_type for c in claims} assert ClaimType.CONDITION_RATING in types assert ClaimType.OBSERVATION in types rating = next(c for c in claims if c.claim_type is ClaimType.CONDITION_RATING) assert "is 2" in rating.claim assert rating.evidence.chunk_id == "c1" assert rating.evidence.page == 12 assert rating.verification.supported is True assert rating.verification.confidence == 1.0 def test_findings_to_atomic_claims_abstains_below_confidence(): from app.extraction.extractor import findings_to_atomic_claims f = SurveyFinding( section="Roofing", element="Main roof", condition_rating="2", finding="The main roof covering is natural slate.", evidence=[EvidenceSpan(chunk_id="c1", text="The main roof covering is natural slate")], ) f.support = SupportLevel.PARTIAL # confidence 0.5 < default min 1.0 assert findings_to_atomic_claims([f], min_confidence=1.0) == [] # Lowering the bar admits them. assert findings_to_atomic_claims([f], min_confidence=0.5)