app/tests/test_extraction_grounding.py

"""Regression tests for the citation-grounded extraction layer.

These tests are the executable success criteria for the anti-hallucination
guarantees. They are fully deterministic and require NO OpenAI key — they
exercise the pure-Python validation/contradiction core that gates the LLM.

Covered failure modes (from the spec's TESTING REQUIREMENTS):
- altered condition ratings
- fabricated materials
- invented locations / entity substitution
- unsupported risk / severity claims
- contradiction generation (rating / condition / operational / duplicate)
- malformed / fabricated sentences (paraphrase with no grounding)
- dropped warranties (positive findings preserved when grounded)
- fabricated monetary totals
- citation to a non-existent chunk
"""

from __future__ import annotations

from app.extraction.citation_validator import (
    span_in_chunk,
    validate_finding,
    validate_findings,
)
from app.extraction.contradiction import audit_contradictions
from app.extraction.schemas import (
    ConditionRating,
    ContradictionKind,
    EvidenceSpan,
    SupportLevel,
    SurveyFinding,
)


def _finding(element: str, rating, text: str, *, chunk_id="c1", span=None) -> SurveyFinding:
    return SurveyFinding(
        section="Roofing",
        element=element,
        condition_rating=rating,
        finding=text,
        evidence=[EvidenceSpan(chunk_id=chunk_id, text=span or text)],
    )


# ── ConditionRating enum: closed set, never fabricated ──────────────────────

def test_rating_enum_coerces_unknown_to_na_not_a_guess():
    assert ConditionRating.coerce("2") is ConditionRating.CR2
    assert ConditionRating.coerce("CR3") is ConditionRating.CR3
    # Garbage must degrade to NA, never to a fabricated severity.
    assert ConditionRating.coerce("urgent") is ConditionRating.NA
    assert ConditionRating.coerce("") is ConditionRating.NA
    assert ConditionRating.coerce(None) is ConditionRating.NA


def test_finding_rating_is_schema_constrained():
    f = SurveyFinding(section="Roofing", element="Ridge", condition_rating="severe", finding="x")
    assert f.condition_rating is ConditionRating.NA


# ── span_in_chunk: verbatim + OCR-drift tolerance, but rejects absent text ──

def test_span_match_verbatim_and_drift():
    chunk = "The main roof covering is natural slate, generally in sound condition."
    assert span_in_chunk("natural slate", chunk)
    assert span_in_chunk("The main roof covering is natural slate", chunk)
    # whitespace/case drift still matches
    assert span_in_chunk("NATURAL   slate", chunk)


def test_span_absent_is_rejected():
    chunk = "The main roof covering is natural slate."
    assert not span_in_chunk("concrete interlocking tiles", chunk)


# ── Altered condition ratings are caught by the contradiction audit ─────────

def test_altered_rating_conflict_detected_and_resolved():
    strong = _finding("Main roof", ConditionRating.CR2, "Slate covering shows slipped tiles.")
    strong.support = SupportLevel.SUPPORTED
    weak = _finding("Main roof", ConditionRating.CR1, "Slate covering shows slipped tiles.")
    weak.support = SupportLevel.PARTIAL
    resolved, reports = audit_contradictions([strong, weak])
    assert len(resolved) == 1
    assert resolved[0].condition_rating is ConditionRating.CR2  # stronger evidence kept
    assert any(r.kind is ContradictionKind.RATING_CONFLICT for r in reports)


# ── Fabricated materials / invented locations: entity substitution ──────────

def test_entity_substitution_is_dropped():
    pool = {"c1": "A London plane tree is located near the rear boundary."}
    # Model swapped the species — must be rejected as unsupported entity.
    bad = SurveyFinding(
        section="Grounds", element="Tree", condition_rating="NA",
        finding="A Lombardy Poplar is located near the rear boundary.",
        evidence=[EvidenceSpan(chunk_id="c1", text="located near the rear boundary")],
    )
    support, violations = validate_finding(bad, pool)
    assert support is SupportLevel.NOT_FOUND
    assert any("entity" in v for v in violations)


def test_correct_entity_is_supported():
    pool = {"c1": "A London plane tree is located near the rear boundary."}
    good = SurveyFinding(
        section="Grounds", element="Tree", condition_rating="NA",
        finding="A London plane tree is located near the rear boundary.",
        evidence=[EvidenceSpan(chunk_id="c1", text="A London plane tree is located near the rear boundary")],
    )
    support, violations = validate_finding(good, pool)
    assert support is SupportLevel.SUPPORTED
    assert violations == []


# ── Fabricated monetary totals / numbers ────────────────────────────────────

def test_fabricated_total_is_dropped():
    pool = {"c1": "Repairs to the parapet are recommended."}
    bad = SurveyFinding(
        section="Roofing", element="Parapet", condition_rating="2",
        finding="Repairs to the parapet are recommended at a cost of £12,500.",
        evidence=[EvidenceSpan(chunk_id="c1", text="Repairs to the parapet are recommended")],
    )
    support, violations = validate_finding(bad, pool)
    assert support is SupportLevel.NOT_FOUND
    assert any("number" in v or "amount" in v for v in violations)


def test_grounded_total_is_preserved():
    pool = {"c1": "Repairs to the parapet are recommended at a cost of £12,500."}
    good = SurveyFinding(
        section="Roofing", element="Parapet", condition_rating="2",
        finding="Repairs to the parapet are recommended at a cost of £12,500.",
        evidence=[EvidenceSpan(chunk_id="c1", text="Repairs to the parapet are recommended at a cost of £12,500")],
    )
    support, _ = validate_finding(good, pool)
    assert support is SupportLevel.SUPPORTED


# ── Unsupported risk / severity escalation ──────────────────────────────────

def test_unsupported_severity_is_dropped():
    pool = {"c1": "There is minor surface staining to the ceiling."}
    bad = SurveyFinding(
        section="Interior", element="Ceiling", condition_rating="2",
        finding="There is minor surface staining to the ceiling, a catastrophic and unsafe defect.",
        evidence=[EvidenceSpan(chunk_id="c1", text="There is minor surface staining to the ceiling")],
    )
    support, violations = validate_finding(bad, pool)
    assert support is SupportLevel.NOT_FOUND
    assert any("severity" in v for v in violations)


def test_severity_allowed_when_in_source():
    pool = {"c1": "The boiler flue is unsafe and must not be used."}
    good = SurveyFinding(
        section="Services", element="Boiler flue", condition_rating="3",
        finding="The boiler flue is unsafe and must not be used.",
        evidence=[EvidenceSpan(chunk_id="c1", text="The boiler flue is unsafe and must not be used")],
    )
    support, _ = validate_finding(good, pool)
    assert support is SupportLevel.SUPPORTED


# ── Citation to a non-existent chunk ────────────────────────────────────────

def test_citation_to_unknown_chunk_is_rejected():
    pool = {"c1": "Slate covering is sound."}
    bad = SurveyFinding(
        section="Roofing", element="Covering", condition_rating="1",
        finding="Slate covering is sound.",
        evidence=[EvidenceSpan(chunk_id="ghost", text="Slate covering is sound")],
    )
    support, violations = validate_finding(bad, pool)
    assert support is SupportLevel.NOT_FOUND
    assert any("unknown chunk_id" in v for v in violations)


# ── Malformed / paraphrased fabrication with no grounding ───────────────────

def test_ungrounded_paraphrase_is_dropped():
    pool = {"c1": "The flat roof is covered in felt."}
    bad = SurveyFinding(
        section="Roofing", element="Flat roof", condition_rating="2",
        finding="Extensive structural movement threatens imminent failure of the dwelling.",
        evidence=[EvidenceSpan(chunk_id="c1", text="The flat roof is covered in felt")],
    )
    support, _ = validate_finding(bad, pool)
    assert support is SupportLevel.NOT_FOUND


# ── Positive findings / warranties preserved when grounded ──────────────────

def test_warranty_preserved():
    pool = {"c1": "The replacement boiler was fitted in 2021 and carries a 10 year manufacturer warranty."}
    good = SurveyFinding(
        section="Services", element="Boiler", condition_rating="1",
        finding="The replacement boiler was fitted in 2021 and carries a 10 year manufacturer warranty.",
        evidence=[EvidenceSpan(
            chunk_id="c1",
            text="The replacement boiler was fitted in 2021 and carries a 10 year manufacturer warranty",
        )],
    )
    support, _ = validate_finding(good, pool)
    assert support is SupportLevel.SUPPORTED


# ── Contradiction: satisfactory vs defective ────────────────────────────────

def test_satisfactory_vs_defective_contradiction():
    a = _finding("Gutters", ConditionRating.NA, "The gutters are in good condition and sound.")
    a.support = SupportLevel.SUPPORTED
    b = _finding("Gutters", ConditionRating.NA, "The gutters are defective and leaking badly.")
    b.support = SupportLevel.NOT_FOUND
    resolved, reports = audit_contradictions([a, b])
    assert len(resolved) == 1
    assert resolved[0] is a  # evidence-stronger side kept
    assert any(r.kind is ContradictionKind.CONDITION for r in reports)


# ── Contradiction: operational vs non-operational ───────────────────────────

def test_operational_contradiction():
    a = _finding("Heating", ConditionRating.NA, "The heating system is fully operational and working.")
    a.support = SupportLevel.SUPPORTED
    b = _finding("Heating", ConditionRating.NA, "The heating system is not operational and out of order.")
    b.support = SupportLevel.PARTIAL
    resolved, reports = audit_contradictions([a, b])
    assert len(resolved) == 1
    assert any(r.kind is ContradictionKind.OPERATIONAL for r in reports)


# ── Duplicate collapse ──────────────────────────────────────────────────────

def test_duplicate_findings_collapsed():
    a = _finding("Chimney", ConditionRating.CR2, "The chimney stack shows perished pointing requiring repair.")
    b = _finding("Chimney", ConditionRating.CR2, "The chimney stack shows perished pointing requiring repair.")
    resolved, reports = audit_contradictions([a, b])
    assert len(resolved) == 1
    assert any(r.kind is ContradictionKind.DUPLICATE for r in reports)


# ── End-to-end gate: mixed batch keeps only grounded, non-conflicting ───────

def test_validate_findings_batch_drops_unsupported():
    pool = {
        "c1": "The main roof is natural slate in sound condition.",
        "c2": "The rear addition roof is covered in felt with no visible defects.",
    }
    findings = [
        SurveyFinding(section="Roofing", element="Main roof", condition_rating="1",
                      finding="The main roof is natural slate in sound condition.",
                      evidence=[EvidenceSpan(chunk_id="c1", text="The main roof is natural slate in sound condition")]),
        SurveyFinding(section="Roofing", element="Rear roof", condition_rating="3",
                      finding="The rear addition roof has collapsed and is deadly.",
                      evidence=[EvidenceSpan(chunk_id="c2", text="The rear addition roof is covered in felt")]),
    ]
    kept, dropped = validate_findings(findings, pool)
    assert len(kept) == 1
    assert kept[0].element == "Main roof"
    assert len(dropped) == 1


"""Section-domain scoping (STEP 6) — prevent cross-section contamination."""


def test_classify_section_aliases_and_keywords():
    from app.extraction.domain_scope import classify_section

    assert classify_section("Roof coverings") == "roofing"
    assert classify_section("Chimney stacks") == "chimney"
    assert classify_section("Rainwater pipes and gutters") == "rainwater"
    assert classify_section("Electricity") == "electrical"
    assert classify_section("About the property") == "general"


def test_classify_text_dominant_domain():
    from app.extraction.domain_scope import classify_text

    assert classify_text("The natural slate roof covering has slipped tiles at the ridge.") == "roofing"
    assert classify_text("The consumer unit lacks RCD protection on the circuits.") == "electrical"
    assert classify_text("This paragraph is generic boilerplate with no domain.") == "general"


def test_scope_chunks_excludes_foreign_domain():
    from app.extraction.domain_scope import scope_chunks

    class _Row:
        def __init__(self, text):
            self.text = text

    chunks = [
        _Row("The slate roof covering is sound at the ridge and eaves."),
        _Row("The drainage manhole and inspection chamber were inspected."),
        _Row("General introductory text about the inspection."),
    ]
    kept = scope_chunks("roofing", chunks)
    texts = [c.text for c in kept]
    assert any("slate roof" in t for t in texts)
    assert any("introductory" in t for t in texts)  # general is admissible
    assert not any("manhole" in t for t in texts)   # drainage excluded


def test_scope_chunks_never_starves_under_strict():
    from app.extraction.domain_scope import scope_chunks

    class _Row:
        def __init__(self, text):
            self.text = text

    # All chunks belong to a different domain -> strict returns originals
    # rather than an empty pool.
    chunks = [_Row("The consumer unit and wiring circuits were inspected.")]
    kept = scope_chunks("roofing", chunks, strict=True)
    assert len(kept) == 1
    kept_nonstrict = scope_chunks("roofing", chunks, strict=False)
    assert kept_nonstrict == []


def test_section_extraction_confidence():
    pool = {"c1": "The main roof is natural slate in sound condition."}
    f = SurveyFinding(section="Roofing", element="Main roof", condition_rating="1",
                      finding="The main roof is natural slate in sound condition.",
                      evidence=[EvidenceSpan(chunk_id="c1", text="The main roof is natural slate in sound condition")])
    kept, _ = validate_findings([f], pool)
    from app.extraction.schemas import SectionExtraction
    sec = SectionExtraction(section="Roofing", findings=kept)
    assert sec.confidence == 1.0


"""Post-generation output validator + abstention (forbidden phrases / metrics)."""


def test_output_validator_flags_forbidden_phrases():
    from app.extraction.output_validator import find_violations, is_clean

    bad = "The roof appears to be defective and overall reliability is questionable."
    v = find_violations(bad, evidence="")
    assert any("appears to" in x for x in v)
    assert any("overall" in x for x in v)
    assert not is_clean(bad)


def test_output_validator_flags_fabricated_percentage_and_score():
    from app.extraction.output_validator import find_violations

    v = find_violations("Authenticity score is high with 87% confidence.", evidence="")
    assert any("percentage" in x for x in v)
    assert any("metric" in x for x in v)


def test_output_validator_allows_terms_present_in_evidence():
    from app.extraction.output_validator import is_clean

    # "unsafe" is permitted because it is verbatim in the evidence.
    text = "The flue is unsafe."
    evidence = "The boiler flue is unsafe and must not be used."
    assert is_clean(text, evidence)


def test_output_validator_clean_text_passes():
    from app.extraction.output_validator import is_clean

    assert is_clean("The main roof covering is natural slate.", evidence="")


def test_abstain_on_low_confidence_and_attribution_failure():
    from app.extraction.output_validator import should_abstain

    assert should_abstain("clean text", "", confidence=0.5, min_confidence=1.0)
    assert should_abstain("clean text", "", evidence_aligned=False)
    assert should_abstain("clean text", "", source_attributed=False)
    assert not should_abstain("The roof is slate.", "", confidence=1.0)


def test_findings_to_atomic_claims_grounded():
    from app.extraction.extractor import findings_to_atomic_claims
    from app.extraction.schemas import ClaimType

    f = SurveyFinding(
        section="Roofing", element="Main roof", condition_rating="2",
        finding="The main roof covering is natural slate with slipped tiles.",
        evidence=[EvidenceSpan(chunk_id="c1", page=12, section_label="Page 12",
                               text="The main roof covering is natural slate with slipped tiles")],
    )
    f.support = SupportLevel.SUPPORTED
    claims = findings_to_atomic_claims([f])
    types = {c.claim_type for c in claims}
    assert ClaimType.CONDITION_RATING in types
    assert ClaimType.OBSERVATION in types
    rating = next(c for c in claims if c.claim_type is ClaimType.CONDITION_RATING)
    assert "is 2" in rating.claim
    assert rating.evidence.chunk_id == "c1"
    assert rating.evidence.page == 12
    assert rating.verification.supported is True
    assert rating.verification.confidence == 1.0


def test_findings_to_atomic_claims_abstains_below_confidence():
    from app.extraction.extractor import findings_to_atomic_claims

    f = SurveyFinding(
        section="Roofing", element="Main roof", condition_rating="2",
        finding="The main roof covering is natural slate.",
        evidence=[EvidenceSpan(chunk_id="c1", text="The main roof covering is natural slate")],
    )
    f.support = SupportLevel.PARTIAL  # confidence 0.5 < default min 1.0
    assert findings_to_atomic_claims([f], min_confidence=1.0) == []
    # Lowering the bar admits them.
    assert findings_to_atomic_claims([f], min_confidence=0.5)