File size: 6,750 Bytes
7880373 35676b4 7880373 35676b4 7880373 35676b4 7880373 35676b4 7880373 35676b4 7880373 35676b4 7880373 35676b4 7880373 35676b4 7880373 35676b4 7880373 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 | """Unit tests for conservative post-synthesis reliability scoring.
Verifies that apply_reliability:
- never promotes transcript corroboration above MEDIUM
- keeps a 10-K MD&A fact at HIGH when not corroborated
- downgrades a Risk Factors fact (heuristic on snippet) to MEDIUM
- keeps a stale lone news fact at LOW and notes it
- appends auto evidence_notes capped at 3
"""
from __future__ import annotations
from datetime import datetime, timedelta
import pytest
from agent.post_synthesis import apply_reliability
def _fact(text, source, snippet, reliability="HIGH", verification_status="VERIFIED"):
return {
"text": text,
"source": source,
"reliability": reliability,
"evidence_snippet": snippet,
"verification_status": verification_status,
}
def test_transcript_corroboration_does_not_promote_above_medium():
"""Cross-source similarity is explanatory, never a reliability promotion."""
brief = {
"filing_date": "2026-04-01",
"bull_points": [
_fact(
"Services revenue accelerated meaningfully",
"transcript",
"services revenue grew twenty four percent driven by subscriptions",
reliability="MEDIUM",
),
],
"what_changed": [
_fact(
"Services segment posted a strong quarter",
"10-Q",
"services revenue increased twenty four percent year over year subscriptions",
reliability="HIGH",
),
],
}
out = apply_reliability(brief)
assert out["bull_points"][0]["reliability"] == "MEDIUM"
notes = out.get("evidence_notes") or []
assert not any("uplift" in n.lower() for n in notes), notes
def test_lone_news_stays_low_and_gets_note():
"""News with no corroboration AND >30d old → LOW + auto note."""
old_date = (datetime.now() - timedelta(days=45)).strftime("%Y-%m-%d")
brief = {
"filing_date": old_date,
"bear_points": [
_fact(
"Analyst downgrade",
"news",
"downgraded to neutral after results citing macro uncertainty",
reliability="MEDIUM",
),
],
}
out = apply_reliability(brief)
assert out["bear_points"][0]["reliability"] == "LOW"
notes = out.get("evidence_notes") or []
assert any("only from news" in n.lower() for n in notes), notes
def test_filing_mda_uncorroborated_stays_high():
"""A 10-K MD&A driver with no peer in another source → stays HIGH."""
brief = {
"filing_date": "2026-04-01",
"mda_summary": {
"drivers": [
_fact(
"Operating leverage from cloud platform",
"10-K",
"operating leverage continued as cloud platform scaled across verticals",
reliability="HIGH",
),
],
"headwinds": [],
"language_shift": "",
"key_quote": _fact(
"We expect continued strength",
"10-K",
"we expect continued strength in our cloud business throughout fiscal year",
reliability="HIGH",
),
},
}
out = apply_reliability(brief)
assert out["mda_summary"]["drivers"][0]["reliability"] == "HIGH"
assert out["mda_summary"]["key_quote"]["reliability"] == "HIGH"
def test_risk_factors_heuristic_downgrades_to_medium():
"""A 10-K fact whose snippet contains 'risk', 'litigation', etc → MEDIUM."""
brief = {
"filing_date": "2026-04-01",
"risks_categorized": [
{
**_fact(
"Cybersecurity exposure remains material",
"10-K",
"cybersecurity risks could materially harm operations litigation exposure remains",
reliability="HIGH",
),
"category": "Cybersecurity",
},
],
}
out = apply_reliability(brief)
assert out["risks_categorized"][0]["reliability"] == "MEDIUM"
notes = out.get("evidence_notes") or []
assert any("risk factors" in n.lower() for n in notes), notes
def test_auto_notes_capped_at_three():
"""Many lone-news facts → only 3 auto notes appended."""
old_date = (datetime.now() - timedelta(days=60)).strftime("%Y-%m-%d")
brief = {
"filing_date": old_date,
"bear_points": [
_fact(f"News claim {i}", "news",
f"unique news content number {i} with several distinctive words here",
reliability="MEDIUM")
for i in range(8)
],
"evidence_notes": [],
}
out = apply_reliability(brief)
notes = out.get("evidence_notes") or []
assert len(notes) <= 3
def test_preserves_existing_evidence_notes():
"""Existing LLM-authored notes are kept, auto notes are appended."""
brief = {
"filing_date": "2026-04-01",
"bull_points": [
_fact("X", "transcript", "services revenue grew twenty four percent driven by",
reliability="MEDIUM"),
],
"what_changed": [
_fact("Y", "10-Q", "services revenue increased twenty four percent year over year",
reliability="HIGH"),
],
"evidence_notes": ["Pre-existing LLM note about X"],
}
out = apply_reliability(brief)
notes = out.get("evidence_notes") or []
assert "Pre-existing LLM note about X" in notes
def test_handles_empty_brief():
out = apply_reliability({})
assert out == {}
def test_handles_none_brief():
assert apply_reliability(None) is None
def test_short_snippet_no_corroboration():
"""Snippet with <4 meaningful tokens cannot corroborate anything."""
brief = {
"filing_date": "2026-04-01",
"bull_points": [
_fact("Short snippet", "transcript", "yes good", reliability="MEDIUM"),
],
"what_changed": [
_fact("Same idea", "10-Q", "yes good results", reliability="HIGH"),
],
}
out = apply_reliability(brief)
# transcript stays MEDIUM (no real corroboration)
assert out["bull_points"][0]["reliability"] == "MEDIUM"
def test_missing_verification_status_is_always_low():
brief = {
"bull_points": [
_fact(
"Unverified filing claim",
"10-K",
"a plausible but unverified filing statement",
verification_status=None,
)
]
}
out = apply_reliability(brief)
assert out["bull_points"][0]["reliability"] == "LOW"
|