import json from agent.evidence import ( NO_VERIFIED_SYNTHESIS_MESSAGE, content_hash, evidence_envelope, evidence_records_from, is_usable_evidence_payload, make_evidence_record, parse_evidence_envelope, verify_brief_evidence, verify_fact, ) from agent.post_synthesis import apply_reliability from agent.schemas import EvidenceRef, SourcedFact def _filing_record(): return make_evidence_record( source="10-Q", content="Revenue grew 5% year over year to $12.0 billion.", document_id="sec:AAPL:0001", chunk_id="mda:0", source_url="https://www.sec.gov/example", as_of="2026-04-30", ) def test_zero_verified_sets_honest_message_and_keeps_fact_level_filtering(): brief = { "what_matters_most": "Unsupported model commentary.", "bull_points": [{ "text": "An unsupported claim.", "source": "10-Q", "reliability": "HIGH", "evidence_snippet": "An unsupported claim.", }], } out = verify_brief_evidence(brief, []) assert out["bull_points"] == [] assert out["status"] == "PARTIAL" assert out["what_matters_most"] == NO_VERIFIED_SYNTHESIS_MESSAGE def test_ids_and_hashes_are_stable_across_whitespace_and_metadata_order(): first = make_evidence_record( source="10-Q", content="Revenue grew 5%. \r\nMargin expanded.", document_id="sec:AAPL:0001", chunk_id="mda:0", metadata={"b": 2, "a": 1}, ) second = make_evidence_record( source="10-Q", content="Revenue grew 5%.\nMargin expanded.", document_id="sec:AAPL:0001", chunk_id="mda:0", metadata={"a": 1, "b": 2}, ) assert first.ref.evidence_id == second.ref.evidence_id assert first.ref.content_hash == second.ref.content_hash assert first.ref.content_hash == content_hash(second.content) def test_noncanonical_ref_source_fails_verification_not_validation(): record = _filing_record() raw = record.ref.model_dump() raw["source"] = "10-Q, transcript" supplied = EvidenceRef.model_validate(raw) fact = { "text": "Revenue grew 5%.", "source": "10-Q", "reliability": "HIGH", "evidence_snippet": "Revenue grew 5% year over year to $12.0 billion.", "evidence_ref": supplied.model_dump(mode="json"), } verify_fact(fact, [record]) assert supplied.source == "10-Q, transcript" assert fact["verification_status"] == "FAILED" assert fact["verification_reason"] == "source_mismatch" assert fact["reliability"] == "LOW" def test_sourced_fact_supports_structured_sources_and_defaults_unverified(): fact = SourcedFact( text="Revenue was $12 billion.", source="metrics", reliability="HIGH", evidence_snippet="Revenue: $12.0B", ) assert fact.source == "metrics" assert fact.verification_status == "UNVERIFIED" assert fact.evidence_ref is None def test_ok_empty_and_error_envelopes_are_machine_readable(): record = _filing_record() ok = parse_evidence_envelope(evidence_envelope(tool="search_filing", records=[record])) empty = parse_evidence_envelope( evidence_envelope(tool="search_filing", status="EMPTY", message="No matches") ) error = parse_evidence_envelope( evidence_envelope(tool="search_filing", status="ERROR", message="boom", error_code="SEARCH_FAILED") ) assert ok["schema"] == "evidence.v1" and ok["status"] == "OK" assert empty["status"] == "EMPTY" and empty["records"] == [] assert error["status"] == "ERROR" and error["error"]["code"] == "SEARCH_FAILED" assert evidence_records_from(json.dumps(ok))[0].ref == record.ref assert is_usable_evidence_payload(ok) is True assert is_usable_evidence_payload(empty) is False def test_verified_fact_requires_matching_id_source_hash_snippet_and_numbers(): record = _filing_record() fact = { "text": "Revenue increased 5% to $12 billion.", "source": "10-Q", "reliability": "HIGH", "evidence_snippet": "Revenue grew 5% year over year to $12.0 billion.", "evidence_ref": record.ref.model_dump(mode="json"), "verification_status": "UNVERIFIED", } verify_fact(fact, [record]) assert fact["verification_status"] == "VERIFIED" def test_truncated_content_hash_fails_verification_per_fact(): record = _filing_record() truncated_ref = record.ref.model_dump(mode="json") truncated_ref["content_hash"] = truncated_ref["content_hash"][:47] fact = { "text": "Revenue increased 5% to $12 billion.", "source": "10-Q", "reliability": "HIGH", "evidence_snippet": "Revenue grew 5% year over year to $12.0 billion.", "evidence_ref": truncated_ref, } verify_fact(fact, [record]) assert fact["verification_status"] == "FAILED" assert fact["verification_reason"] == "content_hash_mismatch" assert fact["reliability"] == "LOW" def test_unsupported_number_fails_closed_to_low(): record = _filing_record() fact = { "text": "Revenue increased 9% to $12 billion.", "source": "10-Q", "reliability": "HIGH", "evidence_snippet": "Revenue grew 5% year over year to $12.0 billion.", "evidence_ref": record.ref.model_dump(mode="json"), } verify_fact(fact, [record]) assert fact["verification_status"] == "FAILED" assert fact["verification_reason"] == "unsupported_numeric_claim" assert fact["reliability"] == "LOW" def test_unhashed_metadata_cannot_supply_a_missing_number(): record = make_evidence_record( source="10-Q", content="Revenue increased year over year.", document_id="sec:AAPL:0003", metadata={"untrusted_claim": "Revenue increased 99%."}, ) fact = { "text": "Revenue increased 99%.", "source": "10-Q", "reliability": "HIGH", "evidence_snippet": "Revenue increased year over year.", "evidence_ref": record.ref.model_dump(mode="json"), } verify_fact(fact, [record]) assert fact["verification_status"] == "FAILED" assert fact["verification_reason"] == "unsupported_numeric_claim" def test_trimmed_verbatim_quote_still_verifies(): # Real Haiku output: the model dropped "the Company's" and truncated the # trailing clause to satisfy the prompt's <=30-word cap, breaking a full # substring match even though the quote is genuinely lifted from the filing. record = make_evidence_record( source="10-Q", content=( "The Company expects these trends to intensify, which may " "materially adversely impact the Company's revenue, costs, gross " "margin, results of operations and financial condition." ), document_id="sec:AAPL:0004", chunk_id="risk_factors:0", ) fact = { "text": "Supply constraints are expected to intensify and pressure margins.", "source": "10-Q", "reliability": "HIGH", "evidence_snippet": ( "The Company expects these trends to intensify, which may " "materially adversely impact revenue, costs, gross margin." ), "evidence_ref": record.ref.model_dump(mode="json"), } verify_fact(fact, [record]) assert fact["verification_status"] == "VERIFIED" def test_fabricated_snippet_still_fails_despite_fuzzy_match(): record = _filing_record() fact = { "text": "Management announced a new CEO and a stock split.", "source": "10-Q", "reliability": "HIGH", "evidence_snippet": "The board appointed a new chief executive and approved a two-for-one split.", "evidence_ref": record.ref.model_dump(mode="json"), } verify_fact(fact, [record]) assert fact["verification_status"] == "FAILED" assert fact["verification_reason"] == "snippet_not_found" def test_metrics_source_skips_literal_snippet_match(): # Metrics tool output is one field per line; combining two fields into a # single narrative snippet (as synthesis naturally does) can never be a # contiguous substring. Numeric support is the correct integrity check. record = make_evidence_record( source="metrics", content="Company: Apple Inc. (AAPL)\n Revenue: $109.4B (YoY: +16.4%)\n Gross Margin: 50.1%", document_id="metrics:AAPL:0001", ) fact = { "text": "Revenue $109.4B, up 16.4% YoY, with gross margin of 50.1%.", "source": "metrics", "reliability": "HIGH", "evidence_snippet": "Revenue: $109.4B (YoY: +16.4%), Gross Margin: 50.1%", "evidence_ref": record.ref.model_dump(mode="json"), } verify_fact(fact, [record]) assert fact["verification_status"] == "VERIFIED" def test_metrics_source_still_rejects_unsupported_numbers(): record = make_evidence_record( source="metrics", content="Company: Apple Inc. (AAPL)\n Revenue: $109.4B (YoY: +16.4%)\n Gross Margin: 50.1%", document_id="metrics:AAPL:0001", ) fact = { "text": "Revenue $999B.", "source": "metrics", "reliability": "HIGH", "evidence_snippet": "Revenue: $999B, an unheard-of figure", "evidence_ref": record.ref.model_dump(mode="json"), } verify_fact(fact, [record]) assert fact["verification_status"] == "FAILED" assert fact["verification_reason"] == "unsupported_numeric_claim" def test_missing_or_tampered_reference_fails_closed(): record = _filing_record() missing = { "text": "Revenue grew.", "source": "10-Q", "reliability": "HIGH", "evidence_snippet": "Revenue grew 5%", } verify_fact(missing, [record]) assert missing["verification_status"] == "UNVERIFIED" assert missing["reliability"] == "LOW" tampered_ref = record.ref.model_dump(mode="json") tampered_ref["content_hash"] = "0" * 64 tampered = { "text": "Revenue grew 5%.", "source": "10-Q", "reliability": "HIGH", "evidence_snippet": "Revenue grew 5%", "evidence_ref": tampered_ref, } verify_fact(tampered, [record]) assert tampered["verification_reason"] == "content_hash_mismatch" assert tampered["reliability"] == "LOW" tampered_locator_ref = record.ref.model_dump(mode="json") tampered_locator_ref["source_url"] = "https://example.invalid/not-the-retrieved-document" tampered_locator = { "text": "Revenue grew 5%.", "source": "10-Q", "reliability": "HIGH", "evidence_snippet": "Revenue grew 5%", "evidence_ref": tampered_locator_ref, } verify_fact(tampered_locator, [record]) assert tampered_locator["verification_reason"] == "locator_mismatch" assert tampered_locator["reliability"] == "LOW" def test_brief_hides_unverified_claims_instead_of_only_downgrading_them(): record = _filing_record() envelope = evidence_envelope(tool="search_filing", records=[record]) brief = { "bull_points": [ { "text": "Revenue grew 5%.", "source": "10-Q", "reliability": "LOW", "evidence_snippet": "Revenue grew 5% year over year", "evidence_ref": record.ref.model_dump(mode="json"), }, { "text": "Margins will double next quarter.", "source": "10-Q", "reliability": "HIGH", "evidence_snippet": "Margins will double", }, ], } out = verify_brief_evidence(brief, [envelope]) assert [fact["text"] for fact in out["bull_points"]] == ["Revenue grew 5%."] assert out["bull_points"][0]["verification_status"] == "VERIFIED" assert out["verification_report"] == { "verified": 1, "unverified": 1, "failed": 0, "removed": 1, } assert out["status"] == "PARTIAL" def test_market_expectations_are_quarantined_without_exact_analyst_reference(): brief = { "market_expectations": { "consensus_eps_est": 999.0, "revision_30d_pct": 42.0, "period_aligned": True, "comparison_allowed": True, "d1_price_reaction_pct": 88.0, "event_aligned": True, "event_comparison_allowed": True, "rationale": "Fabricated beat and rally.", }, } out = verify_brief_evidence(brief, []) market = out["market_expectations"] assert market["consensus_eps_est"] is None assert market["revision_30d_pct"] is None assert market["d1_price_reaction_pct"] is None assert market["period_aligned"] is False assert market["event_comparison_allowed"] is False assert market["verification_status"] == "UNVERIFIED" assert out["status"] == "PARTIAL" def test_market_expectations_are_copied_from_verified_analyst_record(): record = make_evidence_record( source="analyst", document_id="analyst:AAPL:2026-04-30", as_of="2026-04-30", content="""Analyst expectations for AAPL: consensus_eps_est: 2.5000 consensus_rev_est_bn: 10.0000 revision_30d_pct: 4.2000 target_period: Q22026 period_aligned: true comparison_allowed: true alignment_status: EXACT_PROVIDER_PERIOD d1_price_reaction_pct: 3.1000 d5_price_reaction_pct: 5.2000 since_release_price_reaction_pct: 6.4000 event_date: 2026-04-30 event_kind: earnings_release event_timing: after_close event_aligned: true event_comparison_allowed: true price_alignment_status: VERIFIED_EARNINGS_EVENT""", ) brief = { "market_expectations": { "consensus_eps_est": 999.0, "period_aligned": True, "comparison_allowed": True, "d1_price_reaction_pct": 88.0, "event_aligned": True, "event_comparison_allowed": True, "evidence_ref": record.ref.model_dump(mode="json"), "rationale": "Model-authored text must be replaced.", }, } out = verify_brief_evidence( brief, [evidence_envelope(tool="get_analyst_expectations", records=[record])] ) market = out["market_expectations"] assert market["consensus_eps_est"] == 2.5 assert market["revision_30d_pct"] == 4.2 assert market["d1_price_reaction_pct"] == 3.1 assert market["verification_status"] == "VERIFIED" assert market["rationale"].startswith("Verified analyst record:") def test_cross_period_risk_and_guidance_labels_are_suppressed(): record = make_evidence_record( source="10-Q", document_id="sec:AAPL:0002", content=( "Supply constraints could materially affect operations. " "Management expects quarterly revenue between $10 billion and $11 billion." ), ) ref = record.ref.model_dump(mode="json") brief = { "risks_categorized": [{ "category": "Operational", "text": "Supply constraints could materially affect operations.", "source": "10-Q", "reliability": "HIGH", "evidence_snippet": "Supply constraints could materially affect operations.", "evidence_ref": ref, "is_new_this_filing": True, }], "guidance_history": [{ "period": "Q2 2026", "text": "Management expects quarterly revenue between $10 billion and $11 billion.", "source": "10-Q", "reliability": "HIGH", "evidence_snippet": "Management expects quarterly revenue between $10 billion and $11 billion.", "evidence_ref": ref, "actual_result": "Delivered $12 billion.", "verdict": "beat", }], } out = verify_brief_evidence( brief, [evidence_envelope(tool="search_filing", records=[record])] ) assert out["risks_categorized"][0]["verification_status"] == "VERIFIED" assert out["risks_categorized"][0]["is_new_this_filing"] is False assert out["guidance_history"][0]["verification_status"] == "VERIFIED" assert out["guidance_history"][0]["actual_result"] is None assert out["guidance_history"][0]["verdict"] is None def test_brief_verification_then_reliability_uses_verified_status(): record = _filing_record() envelope = evidence_envelope(tool="search_filing", records=[record]) brief = { "filing_date": "2026-04-30", "bull_points": [{ "text": "Revenue grew 5%.", "source": "10-Q", "reliability": "LOW", "evidence_snippet": "Revenue grew 5% year over year", "evidence_ref": record.ref.model_dump(mode="json"), "verification_status": "UNVERIFIED", }], } out = apply_reliability(brief, [envelope]) assert out["bull_points"][0]["verification_status"] == "VERIFIED" assert out["bull_points"][0]["reliability"] == "HIGH" assert out["evidence_coverage"]["status"] == "VERIFIED" def test_explicit_unverified_and_legacy_facts_are_both_fail_closed(): unverified = { "filing_date": "2026-04-30", "bull_points": [{ "text": "Claim", "source": "10-Q", "reliability": "HIGH", "evidence_snippet": "Claim", "verification_status": "UNVERIFIED", }], } assert apply_reliability(unverified)["bull_points"][0]["reliability"] == "LOW" legacy = { "filing_date": "2026-04-30", "bull_points": [{ "text": "Claim", "source": "10-Q", "reliability": "LOW", "evidence_snippet": "Claim", }], } assert apply_reliability(legacy)["bull_points"][0]["reliability"] == "LOW"