fix: allow metrics/analyst sources in CategorizedRisk and GuidancePoint
Browse filesCategorizedRisk.source and GuidancePoint.source hand-rolled a narrower
Literal["10-K", "10-Q", "transcript", "news"] instead of reusing the
shared EvidenceSource type (which also allows "metrics" and
"analyst"). Since get_financial_metrics legitimately tags its
evidence with source="metrics" (guidance figures and risk-relevant
metrics come straight from the ingested XBRL data), any brief where
the model correctly cited a metrics-sourced risk or guidance point
failed BriefOutput validation outright and fell back to an empty
PARTIAL brief — confirmed on a live AAPL run right after the
max_tokens fix (602548d) let synthesis complete for the first time.
ManagementCommentaryTopic.source is intentionally left unchanged
(deliberately narrower — no metrics/news commentary, per prompt and
_drop_news_commentary sanitizer).
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
- agent/schemas.py +2 -2
- tests/test_schemas.py +50 -0
|
@@ -478,7 +478,7 @@ class CategorizedRisk(BaseModel):
|
|
| 478 |
description="Risk category."
|
| 479 |
)
|
| 480 |
text: str = Field(description="The risk, 1-2 sentences grounded in filing language.")
|
| 481 |
-
source:
|
| 482 |
description="Document type where this risk was cited."
|
| 483 |
)
|
| 484 |
reliability: Literal["HIGH", "MEDIUM", "LOW"] = Field(
|
|
@@ -550,7 +550,7 @@ class GuidancePoint(BaseModel):
|
|
| 550 |
|
| 551 |
period: str = Field(description="The filing period when this guidance was given, e.g. 'Q1 2025'.")
|
| 552 |
text: str = Field(description="The guidance statement, 1-2 sentences.")
|
| 553 |
-
source:
|
| 554 |
description="Document type where this guidance appeared."
|
| 555 |
)
|
| 556 |
|
|
|
|
| 478 |
description="Risk category."
|
| 479 |
)
|
| 480 |
text: str = Field(description="The risk, 1-2 sentences grounded in filing language.")
|
| 481 |
+
source: EvidenceSource = Field(
|
| 482 |
description="Document type where this risk was cited."
|
| 483 |
)
|
| 484 |
reliability: Literal["HIGH", "MEDIUM", "LOW"] = Field(
|
|
|
|
| 550 |
|
| 551 |
period: str = Field(description="The filing period when this guidance was given, e.g. 'Q1 2025'.")
|
| 552 |
text: str = Field(description="The guidance statement, 1-2 sentences.")
|
| 553 |
+
source: EvidenceSource = Field(
|
| 554 |
description="Document type where this guidance appeared."
|
| 555 |
)
|
| 556 |
|
|
@@ -99,6 +99,25 @@ def test_brief_output_valid():
|
|
| 99 |
assert brief.what_to_watch == ["Q2 iPhone shipments", "AI feature adoption"]
|
| 100 |
|
| 101 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 102 |
def test_sourced_fact_ignores_extra_fields():
|
| 103 |
# ConfigDict(extra="ignore") — extra fields are silently dropped, not rejected.
|
| 104 |
# This is intentional: LLM output may include unexpected keys.
|
|
@@ -166,6 +185,17 @@ def test_management_commentary_topic_rejects_news_source():
|
|
| 166 |
)
|
| 167 |
|
| 168 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 169 |
def test_management_commentary_topic_trims_long_snippet():
|
| 170 |
snippet = " ".join([f"word{i}" for i in range(40)])
|
| 171 |
t = ManagementCommentaryTopic(
|
|
@@ -228,6 +258,16 @@ def test_categorized_risk_accepts_demand():
|
|
| 228 |
assert r.category == "Demand"
|
| 229 |
|
| 230 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 231 |
def test_categorized_risk_case_folds_lowercase():
|
| 232 |
r = _risk(category="regulatory")
|
| 233 |
assert r.category == "Regulatory"
|
|
@@ -593,6 +633,16 @@ def _gp(**kw) -> GuidancePoint:
|
|
| 593 |
return GuidancePoint(**(defaults | kw))
|
| 594 |
|
| 595 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 596 |
def test_guidance_verdict_valid_passthrough():
|
| 597 |
assert _gp(verdict="beat").verdict == "beat"
|
| 598 |
assert _gp(verdict="in-line").verdict == "in-line"
|
|
|
|
| 99 |
assert brief.what_to_watch == ["Q2 iPhone shipments", "AI feature adoption"]
|
| 100 |
|
| 101 |
|
| 102 |
+
def test_brief_output_accepts_metrics_sources_for_risk_and_guidance():
|
| 103 |
+
brief = _minimal_brief(
|
| 104 |
+
risks_categorized=[dict(
|
| 105 |
+
category="Financial",
|
| 106 |
+
text="Margin compression risk.",
|
| 107 |
+
source="metrics",
|
| 108 |
+
reliability="HIGH",
|
| 109 |
+
is_new_this_filing=False,
|
| 110 |
+
)],
|
| 111 |
+
guidance_history=[dict(
|
| 112 |
+
period="Q3 2025",
|
| 113 |
+
text="Revenue guidance narrowed.",
|
| 114 |
+
source="metrics",
|
| 115 |
+
)],
|
| 116 |
+
)
|
| 117 |
+
assert brief.risks_categorized[0].source == "metrics"
|
| 118 |
+
assert brief.guidance_history[0].source == "metrics"
|
| 119 |
+
|
| 120 |
+
|
| 121 |
def test_sourced_fact_ignores_extra_fields():
|
| 122 |
# ConfigDict(extra="ignore") — extra fields are silently dropped, not rejected.
|
| 123 |
# This is intentional: LLM output may include unexpected keys.
|
|
|
|
| 185 |
)
|
| 186 |
|
| 187 |
|
| 188 |
+
def test_management_commentary_topic_rejects_metrics_source():
|
| 189 |
+
with pytest.raises(ValidationError):
|
| 190 |
+
ManagementCommentaryTopic(
|
| 191 |
+
topic="Revenue guidance",
|
| 192 |
+
summary="Revenue guidance was updated.",
|
| 193 |
+
source="metrics",
|
| 194 |
+
reliability="HIGH",
|
| 195 |
+
evidence_snippet="Revenue guidance was updated.",
|
| 196 |
+
)
|
| 197 |
+
|
| 198 |
+
|
| 199 |
def test_management_commentary_topic_trims_long_snippet():
|
| 200 |
snippet = " ".join([f"word{i}" for i in range(40)])
|
| 201 |
t = ManagementCommentaryTopic(
|
|
|
|
| 258 |
assert r.category == "Demand"
|
| 259 |
|
| 260 |
|
| 261 |
+
def test_categorized_risk_accepts_metrics_source():
|
| 262 |
+
r = _risk(category="Financial", source="metrics")
|
| 263 |
+
assert r.source == "metrics"
|
| 264 |
+
|
| 265 |
+
|
| 266 |
+
def test_categorized_risk_accepts_analyst_source():
|
| 267 |
+
r = _risk(category="Financial", source="analyst")
|
| 268 |
+
assert r.source == "analyst"
|
| 269 |
+
|
| 270 |
+
|
| 271 |
def test_categorized_risk_case_folds_lowercase():
|
| 272 |
r = _risk(category="regulatory")
|
| 273 |
assert r.category == "Regulatory"
|
|
|
|
| 633 |
return GuidancePoint(**(defaults | kw))
|
| 634 |
|
| 635 |
|
| 636 |
+
def test_guidance_point_accepts_metrics_source():
|
| 637 |
+
g = _gp(source="metrics")
|
| 638 |
+
assert g.source == "metrics"
|
| 639 |
+
|
| 640 |
+
|
| 641 |
+
def test_guidance_point_accepts_analyst_source():
|
| 642 |
+
g = _gp(source="analyst")
|
| 643 |
+
assert g.source == "analyst"
|
| 644 |
+
|
| 645 |
+
|
| 646 |
def test_guidance_verdict_valid_passthrough():
|
| 647 |
assert _gp(verdict="beat").verdict == "beat"
|
| 648 |
assert _gp(verdict="in-line").verdict == "in-line"
|