fix: raise synthesis max_tokens to 64000 and surface truncation explicitly
Browse filesThe single-call synthesis (brief + company profile + per-fact
evidence_ref, merged in 8727e56) needs 25-40K output tokens; the
inherited 16384 cap (from the pre-merge two-call split) cut the JSON
mid-object every time (stop_reason=max_tokens confirmed on a live
AAPL run), failing parsing and producing an empty PARTIAL brief
(verified: 0/0) regardless of data quality or freshness. This was
the actual cause behind every "not enough verified evidence" report
throughout this incident, including the original stale NVDA brief.
64000 is claude-haiku-4-5's output ceiling (verified live), with
billing only for tokens actually generated. Also detect
stop_reason/finish_reason == max_tokens/length on the stream and
raise SynthesisTruncatedError, so a future truncation produces an
explicit "truncated by the token limit" reason instead of a generic
JSON-parse failure message.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
- agent/graph.py +28 -2
- tests/test_graph.py +124 -0
|
@@ -362,6 +362,16 @@ def _format_signals_message(signals: list[dict]) -> str:
|
|
| 362 |
MAX_EDGE_SIGNALS = 12
|
| 363 |
MAX_FILING_SIGNALS = 8
|
| 364 |
MAX_TRANSCRIPT_SIGNALS = 6
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 365 |
|
| 366 |
|
| 367 |
def _cap_signals(signals: list[dict], max_total: int = MAX_EDGE_SIGNALS) -> list[dict]:
|
|
@@ -495,7 +505,7 @@ def create_graph(config: Optional[RunConfig] = None):
|
|
| 495 |
|
| 496 |
def synthesis_node(state: AgentState) -> dict:
|
| 497 |
try:
|
| 498 |
-
llm_plain = make_chat_model(cfg, max_tokens=
|
| 499 |
# Main prompt — cached (ephemeral) on Anthropic. Keep this block stable
|
| 500 |
# so the cache hit rate is preserved regardless of the chosen language.
|
| 501 |
lang = state.get("language") or "English"
|
|
@@ -525,10 +535,22 @@ def create_graph(config: Optional[RunConfig] = None):
|
|
| 525 |
HumanMessage(content="Now produce the structured research brief as a JSON object.")
|
| 526 |
]
|
| 527 |
chunks = []
|
|
|
|
| 528 |
for chunk in llm_plain.stream(synthesis_messages):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 529 |
text = chunk.content if isinstance(chunk.content, str) else ""
|
| 530 |
if text:
|
| 531 |
chunks.append(text)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 532 |
raw = "".join(chunks)
|
| 533 |
clean = _extract_json(raw)
|
| 534 |
data = json.loads(clean)
|
|
@@ -596,7 +618,11 @@ def create_graph(config: Optional[RunConfig] = None):
|
|
| 596 |
except Exception as exc:
|
| 597 |
import sys
|
| 598 |
print(f"[synthesis error] {exc}", file=sys.stderr)
|
| 599 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 600 |
return {
|
| 601 |
"brief": partial,
|
| 602 |
"brief_markdown": None,
|
|
|
|
| 362 |
MAX_EDGE_SIGNALS = 12
|
| 363 |
MAX_FILING_SIGNALS = 8
|
| 364 |
MAX_TRANSCRIPT_SIGNALS = 6
|
| 365 |
+
# Output budget for the single-call synthesis (brief + company profile +
|
| 366 |
+
# one full evidence_ref per fact). 16384 was inherited from the pre-merge
|
| 367 |
+
# two-call split and truncated real AAPL briefs mid-JSON (stop_reason=
|
| 368 |
+
# max_tokens at exactly 16384 output tokens). 64000 is the claude-haiku-4-5
|
| 369 |
+
# output ceiling; billing only covers tokens actually generated.
|
| 370 |
+
SYNTHESIS_MAX_TOKENS = 64000
|
| 371 |
+
|
| 372 |
+
|
| 373 |
+
class SynthesisTruncatedError(RuntimeError):
|
| 374 |
+
"""The synthesis stream was cut off by the max_tokens limit."""
|
| 375 |
|
| 376 |
|
| 377 |
def _cap_signals(signals: list[dict], max_total: int = MAX_EDGE_SIGNALS) -> list[dict]:
|
|
|
|
| 505 |
|
| 506 |
def synthesis_node(state: AgentState) -> dict:
|
| 507 |
try:
|
| 508 |
+
llm_plain = make_chat_model(cfg, max_tokens=SYNTHESIS_MAX_TOKENS)
|
| 509 |
# Main prompt — cached (ephemeral) on Anthropic. Keep this block stable
|
| 510 |
# so the cache hit rate is preserved regardless of the chosen language.
|
| 511 |
lang = state.get("language") or "English"
|
|
|
|
| 535 |
HumanMessage(content="Now produce the structured research brief as a JSON object.")
|
| 536 |
]
|
| 537 |
chunks = []
|
| 538 |
+
stop_reason = None
|
| 539 |
for chunk in llm_plain.stream(synthesis_messages):
|
| 540 |
+
metadata = getattr(chunk, "response_metadata", None) or {}
|
| 541 |
+
stop_reason = (
|
| 542 |
+
metadata.get("stop_reason")
|
| 543 |
+
or metadata.get("finish_reason")
|
| 544 |
+
or stop_reason
|
| 545 |
+
)
|
| 546 |
text = chunk.content if isinstance(chunk.content, str) else ""
|
| 547 |
if text:
|
| 548 |
chunks.append(text)
|
| 549 |
+
if stop_reason in ("max_tokens", "length"):
|
| 550 |
+
raise SynthesisTruncatedError(
|
| 551 |
+
"Synthesis output was truncated by the token limit "
|
| 552 |
+
f"(max_tokens={SYNTHESIS_MAX_TOKENS}); the brief JSON was incomplete."
|
| 553 |
+
)
|
| 554 |
raw = "".join(chunks)
|
| 555 |
clean = _extract_json(raw)
|
| 556 |
data = json.loads(clean)
|
|
|
|
| 618 |
except Exception as exc:
|
| 619 |
import sys
|
| 620 |
print(f"[synthesis error] {exc}", file=sys.stderr)
|
| 621 |
+
if isinstance(exc, SynthesisTruncatedError):
|
| 622 |
+
reason = str(exc)
|
| 623 |
+
else:
|
| 624 |
+
reason = f"Synthesis failed validation: {exc}"
|
| 625 |
+
partial = _partial_brief(state, reason)
|
| 626 |
return {
|
| 627 |
"brief": partial,
|
| 628 |
"brief_markdown": None,
|
|
@@ -243,6 +243,130 @@ def test_synthesis_zero_verified_keeps_filtered_brief_not_partial_skeleton(monke
|
|
| 243 |
assert brief["coverage"]
|
| 244 |
|
| 245 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 246 |
def test_partial_brief_falls_back_to_metrics_db(monkeypatch):
|
| 247 |
monkeypatch.setattr(
|
| 248 |
"storage.metrics_db.get_metrics",
|
|
|
|
| 243 |
assert brief["coverage"]
|
| 244 |
|
| 245 |
|
| 246 |
+
def test_synthesis_requests_expanded_max_tokens(monkeypatch):
|
| 247 |
+
fact = {
|
| 248 |
+
"text": "Revenue increased 12%.",
|
| 249 |
+
"source": "10-Q",
|
| 250 |
+
"reliability": "HIGH",
|
| 251 |
+
"evidence_snippet": "Revenue increased 12%.",
|
| 252 |
+
}
|
| 253 |
+
brief_json = json.dumps({
|
| 254 |
+
"ticker": "AAPL",
|
| 255 |
+
"company_name": "Apple Inc.",
|
| 256 |
+
"filing_date": "2026-04-30",
|
| 257 |
+
"what_matters_most": "Unsupported synthesis commentary.",
|
| 258 |
+
"standout_number": fact,
|
| 259 |
+
"what_changed": [],
|
| 260 |
+
"bull_points": [fact],
|
| 261 |
+
"bear_points": [],
|
| 262 |
+
"what_to_watch": ["Watch Q3 gross margin"],
|
| 263 |
+
"trends": [],
|
| 264 |
+
"mda_summary": {
|
| 265 |
+
"drivers": [],
|
| 266 |
+
"headwinds": [],
|
| 267 |
+
"language_shift": "No prior-period comparison was available.",
|
| 268 |
+
"key_quote": fact,
|
| 269 |
+
},
|
| 270 |
+
"risks_categorized": [],
|
| 271 |
+
"management_commentary": [],
|
| 272 |
+
"guidance_history": [],
|
| 273 |
+
"sentiment": None,
|
| 274 |
+
"market_expectations": None,
|
| 275 |
+
})
|
| 276 |
+
|
| 277 |
+
class FakeLLM:
|
| 278 |
+
def bind_tools(self, tools):
|
| 279 |
+
return self
|
| 280 |
+
|
| 281 |
+
def invoke(self, messages):
|
| 282 |
+
return AIMessage(content="done", tool_calls=[])
|
| 283 |
+
|
| 284 |
+
def stream(self, messages):
|
| 285 |
+
yield AIMessage(content=brief_json)
|
| 286 |
+
|
| 287 |
+
calls = []
|
| 288 |
+
|
| 289 |
+
def spy_make_chat_model(*args, **kwargs):
|
| 290 |
+
calls.append(kwargs)
|
| 291 |
+
return FakeLLM()
|
| 292 |
+
|
| 293 |
+
monkeypatch.setattr("analysis.textdiff.compute", lambda ticker: [])
|
| 294 |
+
monkeypatch.setattr("analysis.tone_drift.compute", lambda ticker: [])
|
| 295 |
+
monkeypatch.setattr(
|
| 296 |
+
"agent.company_profile.collect_profile_evidence",
|
| 297 |
+
lambda ticker, include_metrics=True: [],
|
| 298 |
+
)
|
| 299 |
+
monkeypatch.setattr("agent.graph.make_chat_model", spy_make_chat_model)
|
| 300 |
+
|
| 301 |
+
cfg = RunConfig(
|
| 302 |
+
provider="anthropic",
|
| 303 |
+
model="claude-haiku-4-5-20251001",
|
| 304 |
+
api_key="sk-ant-test",
|
| 305 |
+
)
|
| 306 |
+
initial_state = _state([
|
| 307 |
+
HumanMessage(content="Generate a research brief for AAPL."),
|
| 308 |
+
_tool_msg("get_financial_metrics"),
|
| 309 |
+
_tool_msg("search_filing"),
|
| 310 |
+
_tool_msg("search_filing", suffix="2"),
|
| 311 |
+
_tool_msg("search_transcript"),
|
| 312 |
+
], 0)
|
| 313 |
+
|
| 314 |
+
create_graph(cfg).invoke(initial_state)
|
| 315 |
+
|
| 316 |
+
from agent.graph import SYNTHESIS_MAX_TOKENS
|
| 317 |
+
|
| 318 |
+
assert SYNTHESIS_MAX_TOKENS == 64000
|
| 319 |
+
assert any(
|
| 320 |
+
kwargs.get("max_tokens") == SYNTHESIS_MAX_TOKENS
|
| 321 |
+
for kwargs in calls
|
| 322 |
+
)
|
| 323 |
+
|
| 324 |
+
|
| 325 |
+
def test_synthesis_truncation_produces_explicit_partial_reason(monkeypatch):
|
| 326 |
+
class FakeLLM:
|
| 327 |
+
def bind_tools(self, tools):
|
| 328 |
+
return self
|
| 329 |
+
|
| 330 |
+
def invoke(self, messages):
|
| 331 |
+
return AIMessage(content="done", tool_calls=[])
|
| 332 |
+
|
| 333 |
+
def stream(self, messages):
|
| 334 |
+
yield AIMessage(
|
| 335 |
+
content='{"ticker": "AAPL", "company_name": "Apple',
|
| 336 |
+
response_metadata={"stop_reason": "max_tokens"},
|
| 337 |
+
)
|
| 338 |
+
|
| 339 |
+
monkeypatch.setattr("analysis.textdiff.compute", lambda ticker: [])
|
| 340 |
+
monkeypatch.setattr("analysis.tone_drift.compute", lambda ticker: [])
|
| 341 |
+
monkeypatch.setattr(
|
| 342 |
+
"agent.company_profile.collect_profile_evidence",
|
| 343 |
+
lambda ticker, include_metrics=True: [],
|
| 344 |
+
)
|
| 345 |
+
monkeypatch.setattr("agent.graph.make_chat_model", lambda *args, **kwargs: FakeLLM())
|
| 346 |
+
|
| 347 |
+
cfg = RunConfig(
|
| 348 |
+
provider="anthropic",
|
| 349 |
+
model="claude-haiku-4-5-20251001",
|
| 350 |
+
api_key="sk-ant-test",
|
| 351 |
+
)
|
| 352 |
+
initial_state = _state([
|
| 353 |
+
HumanMessage(content="Generate a research brief for AAPL."),
|
| 354 |
+
_tool_msg("get_financial_metrics"),
|
| 355 |
+
_tool_msg("search_filing"),
|
| 356 |
+
_tool_msg("search_filing", suffix="2"),
|
| 357 |
+
_tool_msg("search_transcript"),
|
| 358 |
+
], 0)
|
| 359 |
+
|
| 360 |
+
final = create_graph(cfg).invoke(initial_state)
|
| 361 |
+
brief = final["brief"]
|
| 362 |
+
|
| 363 |
+
assert brief["status"] == "PARTIAL"
|
| 364 |
+
assert "truncated" in brief["evidence_notes"][0]
|
| 365 |
+
assert "token limit" in brief["evidence_notes"][0]
|
| 366 |
+
assert not brief["evidence_notes"][0].startswith("Synthesis failed validation")
|
| 367 |
+
assert "truncated" in final["synthesis_error"]
|
| 368 |
+
|
| 369 |
+
|
| 370 |
def test_partial_brief_falls_back_to_metrics_db(monkeypatch):
|
| 371 |
monkeypatch.setattr(
|
| 372 |
"storage.metrics_db.get_metrics",
|