tests/test_evidence.py

import json

from agent.evidence import (
    NO_VERIFIED_SYNTHESIS_MESSAGE,
    content_hash,
    evidence_envelope,
    evidence_records_from,
    is_usable_evidence_payload,
    make_evidence_record,
    parse_evidence_envelope,
    verify_brief_evidence,
    verify_fact,
)
from agent.post_synthesis import apply_reliability
from agent.schemas import EvidenceRef, SourcedFact


def _filing_record():
    return make_evidence_record(
        source="10-Q",
        content="Revenue grew 5% year over year to $12.0 billion.",
        document_id="sec:AAPL:0001",
        chunk_id="mda:0",
        source_url="https://www.sec.gov/example",
        as_of="2026-04-30",
    )


def test_zero_verified_sets_honest_message_and_keeps_fact_level_filtering():
    brief = {
        "what_matters_most": "Unsupported model commentary.",
        "bull_points": [{
            "text": "An unsupported claim.",
            "source": "10-Q",
            "reliability": "HIGH",
            "evidence_snippet": "An unsupported claim.",
        }],
    }

    out = verify_brief_evidence(brief, [])

    assert out["bull_points"] == []
    assert out["status"] == "PARTIAL"
    assert out["what_matters_most"] == NO_VERIFIED_SYNTHESIS_MESSAGE


def test_ids_and_hashes_are_stable_across_whitespace_and_metadata_order():
    first = make_evidence_record(
        source="10-Q",
        content="Revenue grew 5%.  \r\nMargin expanded.",
        document_id="sec:AAPL:0001",
        chunk_id="mda:0",
        metadata={"b": 2, "a": 1},
    )
    second = make_evidence_record(
        source="10-Q",
        content="Revenue grew 5%.\nMargin expanded.",
        document_id="sec:AAPL:0001",
        chunk_id="mda:0",
        metadata={"a": 1, "b": 2},
    )
    assert first.ref.evidence_id == second.ref.evidence_id
    assert first.ref.content_hash == second.ref.content_hash
    assert first.ref.content_hash == content_hash(second.content)


def test_noncanonical_ref_source_fails_verification_not_validation():
    record = _filing_record()
    raw = record.ref.model_dump()
    raw["source"] = "10-Q, transcript"
    supplied = EvidenceRef.model_validate(raw)
    fact = {
        "text": "Revenue grew 5%.",
        "source": "10-Q",
        "reliability": "HIGH",
        "evidence_snippet": "Revenue grew 5% year over year to $12.0 billion.",
        "evidence_ref": supplied.model_dump(mode="json"),
    }

    verify_fact(fact, [record])

    assert supplied.source == "10-Q, transcript"
    assert fact["verification_status"] == "FAILED"
    assert fact["verification_reason"] == "source_mismatch"
    assert fact["reliability"] == "LOW"


def test_sourced_fact_supports_structured_sources_and_defaults_unverified():
    fact = SourcedFact(
        text="Revenue was $12 billion.",
        source="metrics",
        reliability="HIGH",
        evidence_snippet="Revenue: $12.0B",
    )
    assert fact.source == "metrics"
    assert fact.verification_status == "UNVERIFIED"
    assert fact.evidence_ref is None


def test_ok_empty_and_error_envelopes_are_machine_readable():
    record = _filing_record()
    ok = parse_evidence_envelope(evidence_envelope(tool="search_filing", records=[record]))
    empty = parse_evidence_envelope(
        evidence_envelope(tool="search_filing", status="EMPTY", message="No matches")
    )
    error = parse_evidence_envelope(
        evidence_envelope(tool="search_filing", status="ERROR", message="boom", error_code="SEARCH_FAILED")
    )
    assert ok["schema"] == "evidence.v1" and ok["status"] == "OK"
    assert empty["status"] == "EMPTY" and empty["records"] == []
    assert error["status"] == "ERROR" and error["error"]["code"] == "SEARCH_FAILED"
    assert evidence_records_from(json.dumps(ok))[0].ref == record.ref
    assert is_usable_evidence_payload(ok) is True
    assert is_usable_evidence_payload(empty) is False


def test_verified_fact_requires_matching_id_source_hash_snippet_and_numbers():
    record = _filing_record()
    fact = {
        "text": "Revenue increased 5% to $12 billion.",
        "source": "10-Q",
        "reliability": "HIGH",
        "evidence_snippet": "Revenue grew 5% year over year to $12.0 billion.",
        "evidence_ref": record.ref.model_dump(mode="json"),
        "verification_status": "UNVERIFIED",
    }
    verify_fact(fact, [record])
    assert fact["verification_status"] == "VERIFIED"


def test_truncated_content_hash_fails_verification_per_fact():
    record = _filing_record()
    truncated_ref = record.ref.model_dump(mode="json")
    truncated_ref["content_hash"] = truncated_ref["content_hash"][:47]
    fact = {
        "text": "Revenue increased 5% to $12 billion.",
        "source": "10-Q",
        "reliability": "HIGH",
        "evidence_snippet": "Revenue grew 5% year over year to $12.0 billion.",
        "evidence_ref": truncated_ref,
    }

    verify_fact(fact, [record])

    assert fact["verification_status"] == "FAILED"
    assert fact["verification_reason"] == "content_hash_mismatch"
    assert fact["reliability"] == "LOW"


def test_unsupported_number_fails_closed_to_low():
    record = _filing_record()
    fact = {
        "text": "Revenue increased 9% to $12 billion.",
        "source": "10-Q",
        "reliability": "HIGH",
        "evidence_snippet": "Revenue grew 5% year over year to $12.0 billion.",
        "evidence_ref": record.ref.model_dump(mode="json"),
    }
    verify_fact(fact, [record])
    assert fact["verification_status"] == "FAILED"
    assert fact["verification_reason"] == "unsupported_numeric_claim"
    assert fact["reliability"] == "LOW"


def test_unhashed_metadata_cannot_supply_a_missing_number():
    record = make_evidence_record(
        source="10-Q",
        content="Revenue increased year over year.",
        document_id="sec:AAPL:0003",
        metadata={"untrusted_claim": "Revenue increased 99%."},
    )
    fact = {
        "text": "Revenue increased 99%.",
        "source": "10-Q",
        "reliability": "HIGH",
        "evidence_snippet": "Revenue increased year over year.",
        "evidence_ref": record.ref.model_dump(mode="json"),
    }

    verify_fact(fact, [record])

    assert fact["verification_status"] == "FAILED"
    assert fact["verification_reason"] == "unsupported_numeric_claim"


def test_trimmed_verbatim_quote_still_verifies():
    # Real Haiku output: the model dropped "the Company's" and truncated the
    # trailing clause to satisfy the prompt's <=30-word cap, breaking a full
    # substring match even though the quote is genuinely lifted from the filing.
    record = make_evidence_record(
        source="10-Q",
        content=(
            "The Company expects these trends to intensify, which may "
            "materially adversely impact the Company's revenue, costs, gross "
            "margin, results of operations and financial condition."
        ),
        document_id="sec:AAPL:0004",
        chunk_id="risk_factors:0",
    )
    fact = {
        "text": "Supply constraints are expected to intensify and pressure margins.",
        "source": "10-Q",
        "reliability": "HIGH",
        "evidence_snippet": (
            "The Company expects these trends to intensify, which may "
            "materially adversely impact revenue, costs, gross margin."
        ),
        "evidence_ref": record.ref.model_dump(mode="json"),
    }

    verify_fact(fact, [record])

    assert fact["verification_status"] == "VERIFIED"


def test_fabricated_snippet_still_fails_despite_fuzzy_match():
    record = _filing_record()
    fact = {
        "text": "Management announced a new CEO and a stock split.",
        "source": "10-Q",
        "reliability": "HIGH",
        "evidence_snippet": "The board appointed a new chief executive and approved a two-for-one split.",
        "evidence_ref": record.ref.model_dump(mode="json"),
    }

    verify_fact(fact, [record])

    assert fact["verification_status"] == "FAILED"
    assert fact["verification_reason"] == "snippet_not_found"


def test_metrics_source_skips_literal_snippet_match():
    # Metrics tool output is one field per line; combining two fields into a
    # single narrative snippet (as synthesis naturally does) can never be a
    # contiguous substring. Numeric support is the correct integrity check.
    record = make_evidence_record(
        source="metrics",
        content="Company: Apple Inc. (AAPL)\n  Revenue: $109.4B (YoY: +16.4%)\n  Gross Margin: 50.1%",
        document_id="metrics:AAPL:0001",
    )
    fact = {
        "text": "Revenue $109.4B, up 16.4% YoY, with gross margin of 50.1%.",
        "source": "metrics",
        "reliability": "HIGH",
        "evidence_snippet": "Revenue: $109.4B (YoY: +16.4%), Gross Margin: 50.1%",
        "evidence_ref": record.ref.model_dump(mode="json"),
    }

    verify_fact(fact, [record])

    assert fact["verification_status"] == "VERIFIED"


def test_metrics_source_still_rejects_unsupported_numbers():
    record = make_evidence_record(
        source="metrics",
        content="Company: Apple Inc. (AAPL)\n  Revenue: $109.4B (YoY: +16.4%)\n  Gross Margin: 50.1%",
        document_id="metrics:AAPL:0001",
    )
    fact = {
        "text": "Revenue $999B.",
        "source": "metrics",
        "reliability": "HIGH",
        "evidence_snippet": "Revenue: $999B, an unheard-of figure",
        "evidence_ref": record.ref.model_dump(mode="json"),
    }

    verify_fact(fact, [record])

    assert fact["verification_status"] == "FAILED"
    assert fact["verification_reason"] == "unsupported_numeric_claim"


def test_missing_or_tampered_reference_fails_closed():
    record = _filing_record()
    missing = {
        "text": "Revenue grew.", "source": "10-Q", "reliability": "HIGH",
        "evidence_snippet": "Revenue grew 5%",
    }
    verify_fact(missing, [record])
    assert missing["verification_status"] == "UNVERIFIED"
    assert missing["reliability"] == "LOW"

    tampered_ref = record.ref.model_dump(mode="json")
    tampered_ref["content_hash"] = "0" * 64
    tampered = {
        "text": "Revenue grew 5%.", "source": "10-Q", "reliability": "HIGH",
        "evidence_snippet": "Revenue grew 5%", "evidence_ref": tampered_ref,
    }
    verify_fact(tampered, [record])
    assert tampered["verification_reason"] == "content_hash_mismatch"
    assert tampered["reliability"] == "LOW"

    tampered_locator_ref = record.ref.model_dump(mode="json")
    tampered_locator_ref["source_url"] = "https://example.invalid/not-the-retrieved-document"
    tampered_locator = {
        "text": "Revenue grew 5%.", "source": "10-Q", "reliability": "HIGH",
        "evidence_snippet": "Revenue grew 5%", "evidence_ref": tampered_locator_ref,
    }
    verify_fact(tampered_locator, [record])
    assert tampered_locator["verification_reason"] == "locator_mismatch"
    assert tampered_locator["reliability"] == "LOW"


def test_brief_hides_unverified_claims_instead_of_only_downgrading_them():
    record = _filing_record()
    envelope = evidence_envelope(tool="search_filing", records=[record])
    brief = {
        "bull_points": [
            {
                "text": "Revenue grew 5%.",
                "source": "10-Q",
                "reliability": "LOW",
                "evidence_snippet": "Revenue grew 5% year over year",
                "evidence_ref": record.ref.model_dump(mode="json"),
            },
            {
                "text": "Margins will double next quarter.",
                "source": "10-Q",
                "reliability": "HIGH",
                "evidence_snippet": "Margins will double",
            },
        ],
    }

    out = verify_brief_evidence(brief, [envelope])

    assert [fact["text"] for fact in out["bull_points"]] == ["Revenue grew 5%."]
    assert out["bull_points"][0]["verification_status"] == "VERIFIED"
    assert out["verification_report"] == {
        "verified": 1, "unverified": 1, "failed": 0, "removed": 1,
    }
    assert out["status"] == "PARTIAL"


def test_market_expectations_are_quarantined_without_exact_analyst_reference():
    brief = {
        "market_expectations": {
            "consensus_eps_est": 999.0,
            "revision_30d_pct": 42.0,
            "period_aligned": True,
            "comparison_allowed": True,
            "d1_price_reaction_pct": 88.0,
            "event_aligned": True,
            "event_comparison_allowed": True,
            "rationale": "Fabricated beat and rally.",
        },
    }

    out = verify_brief_evidence(brief, [])
    market = out["market_expectations"]
    assert market["consensus_eps_est"] is None
    assert market["revision_30d_pct"] is None
    assert market["d1_price_reaction_pct"] is None
    assert market["period_aligned"] is False
    assert market["event_comparison_allowed"] is False
    assert market["verification_status"] == "UNVERIFIED"
    assert out["status"] == "PARTIAL"


def test_market_expectations_are_copied_from_verified_analyst_record():
    record = make_evidence_record(
        source="analyst",
        document_id="analyst:AAPL:2026-04-30",
        as_of="2026-04-30",
        content="""Analyst expectations for AAPL:
consensus_eps_est: 2.5000
consensus_rev_est_bn: 10.0000
revision_30d_pct: 4.2000
target_period: Q22026
period_aligned: true
comparison_allowed: true
alignment_status: EXACT_PROVIDER_PERIOD
d1_price_reaction_pct: 3.1000
d5_price_reaction_pct: 5.2000
since_release_price_reaction_pct: 6.4000
event_date: 2026-04-30
event_kind: earnings_release
event_timing: after_close
event_aligned: true
event_comparison_allowed: true
price_alignment_status: VERIFIED_EARNINGS_EVENT""",
    )
    brief = {
        "market_expectations": {
            "consensus_eps_est": 999.0,
            "period_aligned": True,
            "comparison_allowed": True,
            "d1_price_reaction_pct": 88.0,
            "event_aligned": True,
            "event_comparison_allowed": True,
            "evidence_ref": record.ref.model_dump(mode="json"),
            "rationale": "Model-authored text must be replaced.",
        },
    }

    out = verify_brief_evidence(
        brief, [evidence_envelope(tool="get_analyst_expectations", records=[record])]
    )
    market = out["market_expectations"]
    assert market["consensus_eps_est"] == 2.5
    assert market["revision_30d_pct"] == 4.2
    assert market["d1_price_reaction_pct"] == 3.1
    assert market["verification_status"] == "VERIFIED"
    assert market["rationale"].startswith("Verified analyst record:")


def test_cross_period_risk_and_guidance_labels_are_suppressed():
    record = make_evidence_record(
        source="10-Q",
        document_id="sec:AAPL:0002",
        content=(
            "Supply constraints could materially affect operations. "
            "Management expects quarterly revenue between $10 billion and $11 billion."
        ),
    )
    ref = record.ref.model_dump(mode="json")
    brief = {
        "risks_categorized": [{
            "category": "Operational",
            "text": "Supply constraints could materially affect operations.",
            "source": "10-Q",
            "reliability": "HIGH",
            "evidence_snippet": "Supply constraints could materially affect operations.",
            "evidence_ref": ref,
            "is_new_this_filing": True,
        }],
        "guidance_history": [{
            "period": "Q2 2026",
            "text": "Management expects quarterly revenue between $10 billion and $11 billion.",
            "source": "10-Q",
            "reliability": "HIGH",
            "evidence_snippet": "Management expects quarterly revenue between $10 billion and $11 billion.",
            "evidence_ref": ref,
            "actual_result": "Delivered $12 billion.",
            "verdict": "beat",
        }],
    }

    out = verify_brief_evidence(
        brief, [evidence_envelope(tool="search_filing", records=[record])]
    )
    assert out["risks_categorized"][0]["verification_status"] == "VERIFIED"
    assert out["risks_categorized"][0]["is_new_this_filing"] is False
    assert out["guidance_history"][0]["verification_status"] == "VERIFIED"
    assert out["guidance_history"][0]["actual_result"] is None
    assert out["guidance_history"][0]["verdict"] is None


def test_brief_verification_then_reliability_uses_verified_status():
    record = _filing_record()
    envelope = evidence_envelope(tool="search_filing", records=[record])
    brief = {
        "filing_date": "2026-04-30",
        "bull_points": [{
            "text": "Revenue grew 5%.",
            "source": "10-Q",
            "reliability": "LOW",
            "evidence_snippet": "Revenue grew 5% year over year",
            "evidence_ref": record.ref.model_dump(mode="json"),
            "verification_status": "UNVERIFIED",
        }],
    }
    out = apply_reliability(brief, [envelope])
    assert out["bull_points"][0]["verification_status"] == "VERIFIED"
    assert out["bull_points"][0]["reliability"] == "HIGH"
    assert out["evidence_coverage"]["status"] == "VERIFIED"


def test_explicit_unverified_and_legacy_facts_are_both_fail_closed():
    unverified = {
        "filing_date": "2026-04-30",
        "bull_points": [{
            "text": "Claim", "source": "10-Q", "reliability": "HIGH",
            "evidence_snippet": "Claim", "verification_status": "UNVERIFIED",
        }],
    }
    assert apply_reliability(unverified)["bull_points"][0]["reliability"] == "LOW"

    legacy = {
        "filing_date": "2026-04-30",
        "bull_points": [{
            "text": "Claim", "source": "10-Q", "reliability": "LOW",
            "evidence_snippet": "Claim",
        }],
    }
    assert apply_reliability(legacy)["bull_points"][0]["reliability"] == "LOW"