Viney Claude Sonnet 5 commited on
Commit
e526b3a
·
1 Parent(s): 602548d

fix: allow metrics/analyst sources in CategorizedRisk and GuidancePoint

Browse files

CategorizedRisk.source and GuidancePoint.source hand-rolled a narrower
Literal["10-K", "10-Q", "transcript", "news"] instead of reusing the
shared EvidenceSource type (which also allows "metrics" and
"analyst"). Since get_financial_metrics legitimately tags its
evidence with source="metrics" (guidance figures and risk-relevant
metrics come straight from the ingested XBRL data), any brief where
the model correctly cited a metrics-sourced risk or guidance point
failed BriefOutput validation outright and fell back to an empty
PARTIAL brief — confirmed on a live AAPL run right after the
max_tokens fix (602548d) let synthesis complete for the first time.

ManagementCommentaryTopic.source is intentionally left unchanged
(deliberately narrower — no metrics/news commentary, per prompt and
_drop_news_commentary sanitizer).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

Files changed (2) hide show
  1. agent/schemas.py +2 -2
  2. tests/test_schemas.py +50 -0
agent/schemas.py CHANGED
@@ -478,7 +478,7 @@ class CategorizedRisk(BaseModel):
478
  description="Risk category."
479
  )
480
  text: str = Field(description="The risk, 1-2 sentences grounded in filing language.")
481
- source: Literal["10-K", "10-Q", "transcript", "news"] = Field(
482
  description="Document type where this risk was cited."
483
  )
484
  reliability: Literal["HIGH", "MEDIUM", "LOW"] = Field(
@@ -550,7 +550,7 @@ class GuidancePoint(BaseModel):
550
 
551
  period: str = Field(description="The filing period when this guidance was given, e.g. 'Q1 2025'.")
552
  text: str = Field(description="The guidance statement, 1-2 sentences.")
553
- source: Literal["10-K", "10-Q", "transcript", "news"] = Field(
554
  description="Document type where this guidance appeared."
555
  )
556
 
 
478
  description="Risk category."
479
  )
480
  text: str = Field(description="The risk, 1-2 sentences grounded in filing language.")
481
+ source: EvidenceSource = Field(
482
  description="Document type where this risk was cited."
483
  )
484
  reliability: Literal["HIGH", "MEDIUM", "LOW"] = Field(
 
550
 
551
  period: str = Field(description="The filing period when this guidance was given, e.g. 'Q1 2025'.")
552
  text: str = Field(description="The guidance statement, 1-2 sentences.")
553
+ source: EvidenceSource = Field(
554
  description="Document type where this guidance appeared."
555
  )
556
 
tests/test_schemas.py CHANGED
@@ -99,6 +99,25 @@ def test_brief_output_valid():
99
  assert brief.what_to_watch == ["Q2 iPhone shipments", "AI feature adoption"]
100
 
101
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
102
  def test_sourced_fact_ignores_extra_fields():
103
  # ConfigDict(extra="ignore") — extra fields are silently dropped, not rejected.
104
  # This is intentional: LLM output may include unexpected keys.
@@ -166,6 +185,17 @@ def test_management_commentary_topic_rejects_news_source():
166
  )
167
 
168
 
 
 
 
 
 
 
 
 
 
 
 
169
  def test_management_commentary_topic_trims_long_snippet():
170
  snippet = " ".join([f"word{i}" for i in range(40)])
171
  t = ManagementCommentaryTopic(
@@ -228,6 +258,16 @@ def test_categorized_risk_accepts_demand():
228
  assert r.category == "Demand"
229
 
230
 
 
 
 
 
 
 
 
 
 
 
231
  def test_categorized_risk_case_folds_lowercase():
232
  r = _risk(category="regulatory")
233
  assert r.category == "Regulatory"
@@ -593,6 +633,16 @@ def _gp(**kw) -> GuidancePoint:
593
  return GuidancePoint(**(defaults | kw))
594
 
595
 
 
 
 
 
 
 
 
 
 
 
596
  def test_guidance_verdict_valid_passthrough():
597
  assert _gp(verdict="beat").verdict == "beat"
598
  assert _gp(verdict="in-line").verdict == "in-line"
 
99
  assert brief.what_to_watch == ["Q2 iPhone shipments", "AI feature adoption"]
100
 
101
 
102
+ def test_brief_output_accepts_metrics_sources_for_risk_and_guidance():
103
+ brief = _minimal_brief(
104
+ risks_categorized=[dict(
105
+ category="Financial",
106
+ text="Margin compression risk.",
107
+ source="metrics",
108
+ reliability="HIGH",
109
+ is_new_this_filing=False,
110
+ )],
111
+ guidance_history=[dict(
112
+ period="Q3 2025",
113
+ text="Revenue guidance narrowed.",
114
+ source="metrics",
115
+ )],
116
+ )
117
+ assert brief.risks_categorized[0].source == "metrics"
118
+ assert brief.guidance_history[0].source == "metrics"
119
+
120
+
121
  def test_sourced_fact_ignores_extra_fields():
122
  # ConfigDict(extra="ignore") — extra fields are silently dropped, not rejected.
123
  # This is intentional: LLM output may include unexpected keys.
 
185
  )
186
 
187
 
188
+ def test_management_commentary_topic_rejects_metrics_source():
189
+ with pytest.raises(ValidationError):
190
+ ManagementCommentaryTopic(
191
+ topic="Revenue guidance",
192
+ summary="Revenue guidance was updated.",
193
+ source="metrics",
194
+ reliability="HIGH",
195
+ evidence_snippet="Revenue guidance was updated.",
196
+ )
197
+
198
+
199
  def test_management_commentary_topic_trims_long_snippet():
200
  snippet = " ".join([f"word{i}" for i in range(40)])
201
  t = ManagementCommentaryTopic(
 
258
  assert r.category == "Demand"
259
 
260
 
261
+ def test_categorized_risk_accepts_metrics_source():
262
+ r = _risk(category="Financial", source="metrics")
263
+ assert r.source == "metrics"
264
+
265
+
266
+ def test_categorized_risk_accepts_analyst_source():
267
+ r = _risk(category="Financial", source="analyst")
268
+ assert r.source == "analyst"
269
+
270
+
271
  def test_categorized_risk_case_folds_lowercase():
272
  r = _risk(category="regulatory")
273
  assert r.category == "Regulatory"
 
633
  return GuidancePoint(**(defaults | kw))
634
 
635
 
636
+ def test_guidance_point_accepts_metrics_source():
637
+ g = _gp(source="metrics")
638
+ assert g.source == "metrics"
639
+
640
+
641
+ def test_guidance_point_accepts_analyst_source():
642
+ g = _gp(source="analyst")
643
+ assert g.source == "analyst"
644
+
645
+
646
  def test_guidance_verdict_valid_passthrough():
647
  assert _gp(verdict="beat").verdict == "beat"
648
  assert _gp(verdict="in-line").verdict == "in-line"