Viney Claude Sonnet 5 commited on
Commit
602548d
·
1 Parent(s): 273c984

fix: raise synthesis max_tokens to 64000 and surface truncation explicitly

Browse files

The single-call synthesis (brief + company profile + per-fact
evidence_ref, merged in 8727e56) needs 25-40K output tokens; the
inherited 16384 cap (from the pre-merge two-call split) cut the JSON
mid-object every time (stop_reason=max_tokens confirmed on a live
AAPL run), failing parsing and producing an empty PARTIAL brief
(verified: 0/0) regardless of data quality or freshness. This was
the actual cause behind every "not enough verified evidence" report
throughout this incident, including the original stale NVDA brief.

64000 is claude-haiku-4-5's output ceiling (verified live), with
billing only for tokens actually generated. Also detect
stop_reason/finish_reason == max_tokens/length on the stream and
raise SynthesisTruncatedError, so a future truncation produces an
explicit "truncated by the token limit" reason instead of a generic
JSON-parse failure message.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

Files changed (2) hide show
  1. agent/graph.py +28 -2
  2. tests/test_graph.py +124 -0
agent/graph.py CHANGED
@@ -362,6 +362,16 @@ def _format_signals_message(signals: list[dict]) -> str:
362
  MAX_EDGE_SIGNALS = 12
363
  MAX_FILING_SIGNALS = 8
364
  MAX_TRANSCRIPT_SIGNALS = 6
 
 
 
 
 
 
 
 
 
 
365
 
366
 
367
  def _cap_signals(signals: list[dict], max_total: int = MAX_EDGE_SIGNALS) -> list[dict]:
@@ -495,7 +505,7 @@ def create_graph(config: Optional[RunConfig] = None):
495
 
496
  def synthesis_node(state: AgentState) -> dict:
497
  try:
498
- llm_plain = make_chat_model(cfg, max_tokens=16384)
499
  # Main prompt — cached (ephemeral) on Anthropic. Keep this block stable
500
  # so the cache hit rate is preserved regardless of the chosen language.
501
  lang = state.get("language") or "English"
@@ -525,10 +535,22 @@ def create_graph(config: Optional[RunConfig] = None):
525
  HumanMessage(content="Now produce the structured research brief as a JSON object.")
526
  ]
527
  chunks = []
 
528
  for chunk in llm_plain.stream(synthesis_messages):
 
 
 
 
 
 
529
  text = chunk.content if isinstance(chunk.content, str) else ""
530
  if text:
531
  chunks.append(text)
 
 
 
 
 
532
  raw = "".join(chunks)
533
  clean = _extract_json(raw)
534
  data = json.loads(clean)
@@ -596,7 +618,11 @@ def create_graph(config: Optional[RunConfig] = None):
596
  except Exception as exc:
597
  import sys
598
  print(f"[synthesis error] {exc}", file=sys.stderr)
599
- partial = _partial_brief(state, f"Synthesis failed validation: {exc}")
 
 
 
 
600
  return {
601
  "brief": partial,
602
  "brief_markdown": None,
 
362
  MAX_EDGE_SIGNALS = 12
363
  MAX_FILING_SIGNALS = 8
364
  MAX_TRANSCRIPT_SIGNALS = 6
365
+ # Output budget for the single-call synthesis (brief + company profile +
366
+ # one full evidence_ref per fact). 16384 was inherited from the pre-merge
367
+ # two-call split and truncated real AAPL briefs mid-JSON (stop_reason=
368
+ # max_tokens at exactly 16384 output tokens). 64000 is the claude-haiku-4-5
369
+ # output ceiling; billing only covers tokens actually generated.
370
+ SYNTHESIS_MAX_TOKENS = 64000
371
+
372
+
373
+ class SynthesisTruncatedError(RuntimeError):
374
+ """The synthesis stream was cut off by the max_tokens limit."""
375
 
376
 
377
  def _cap_signals(signals: list[dict], max_total: int = MAX_EDGE_SIGNALS) -> list[dict]:
 
505
 
506
  def synthesis_node(state: AgentState) -> dict:
507
  try:
508
+ llm_plain = make_chat_model(cfg, max_tokens=SYNTHESIS_MAX_TOKENS)
509
  # Main prompt — cached (ephemeral) on Anthropic. Keep this block stable
510
  # so the cache hit rate is preserved regardless of the chosen language.
511
  lang = state.get("language") or "English"
 
535
  HumanMessage(content="Now produce the structured research brief as a JSON object.")
536
  ]
537
  chunks = []
538
+ stop_reason = None
539
  for chunk in llm_plain.stream(synthesis_messages):
540
+ metadata = getattr(chunk, "response_metadata", None) or {}
541
+ stop_reason = (
542
+ metadata.get("stop_reason")
543
+ or metadata.get("finish_reason")
544
+ or stop_reason
545
+ )
546
  text = chunk.content if isinstance(chunk.content, str) else ""
547
  if text:
548
  chunks.append(text)
549
+ if stop_reason in ("max_tokens", "length"):
550
+ raise SynthesisTruncatedError(
551
+ "Synthesis output was truncated by the token limit "
552
+ f"(max_tokens={SYNTHESIS_MAX_TOKENS}); the brief JSON was incomplete."
553
+ )
554
  raw = "".join(chunks)
555
  clean = _extract_json(raw)
556
  data = json.loads(clean)
 
618
  except Exception as exc:
619
  import sys
620
  print(f"[synthesis error] {exc}", file=sys.stderr)
621
+ if isinstance(exc, SynthesisTruncatedError):
622
+ reason = str(exc)
623
+ else:
624
+ reason = f"Synthesis failed validation: {exc}"
625
+ partial = _partial_brief(state, reason)
626
  return {
627
  "brief": partial,
628
  "brief_markdown": None,
tests/test_graph.py CHANGED
@@ -243,6 +243,130 @@ def test_synthesis_zero_verified_keeps_filtered_brief_not_partial_skeleton(monke
243
  assert brief["coverage"]
244
 
245
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
246
  def test_partial_brief_falls_back_to_metrics_db(monkeypatch):
247
  monkeypatch.setattr(
248
  "storage.metrics_db.get_metrics",
 
243
  assert brief["coverage"]
244
 
245
 
246
+ def test_synthesis_requests_expanded_max_tokens(monkeypatch):
247
+ fact = {
248
+ "text": "Revenue increased 12%.",
249
+ "source": "10-Q",
250
+ "reliability": "HIGH",
251
+ "evidence_snippet": "Revenue increased 12%.",
252
+ }
253
+ brief_json = json.dumps({
254
+ "ticker": "AAPL",
255
+ "company_name": "Apple Inc.",
256
+ "filing_date": "2026-04-30",
257
+ "what_matters_most": "Unsupported synthesis commentary.",
258
+ "standout_number": fact,
259
+ "what_changed": [],
260
+ "bull_points": [fact],
261
+ "bear_points": [],
262
+ "what_to_watch": ["Watch Q3 gross margin"],
263
+ "trends": [],
264
+ "mda_summary": {
265
+ "drivers": [],
266
+ "headwinds": [],
267
+ "language_shift": "No prior-period comparison was available.",
268
+ "key_quote": fact,
269
+ },
270
+ "risks_categorized": [],
271
+ "management_commentary": [],
272
+ "guidance_history": [],
273
+ "sentiment": None,
274
+ "market_expectations": None,
275
+ })
276
+
277
+ class FakeLLM:
278
+ def bind_tools(self, tools):
279
+ return self
280
+
281
+ def invoke(self, messages):
282
+ return AIMessage(content="done", tool_calls=[])
283
+
284
+ def stream(self, messages):
285
+ yield AIMessage(content=brief_json)
286
+
287
+ calls = []
288
+
289
+ def spy_make_chat_model(*args, **kwargs):
290
+ calls.append(kwargs)
291
+ return FakeLLM()
292
+
293
+ monkeypatch.setattr("analysis.textdiff.compute", lambda ticker: [])
294
+ monkeypatch.setattr("analysis.tone_drift.compute", lambda ticker: [])
295
+ monkeypatch.setattr(
296
+ "agent.company_profile.collect_profile_evidence",
297
+ lambda ticker, include_metrics=True: [],
298
+ )
299
+ monkeypatch.setattr("agent.graph.make_chat_model", spy_make_chat_model)
300
+
301
+ cfg = RunConfig(
302
+ provider="anthropic",
303
+ model="claude-haiku-4-5-20251001",
304
+ api_key="sk-ant-test",
305
+ )
306
+ initial_state = _state([
307
+ HumanMessage(content="Generate a research brief for AAPL."),
308
+ _tool_msg("get_financial_metrics"),
309
+ _tool_msg("search_filing"),
310
+ _tool_msg("search_filing", suffix="2"),
311
+ _tool_msg("search_transcript"),
312
+ ], 0)
313
+
314
+ create_graph(cfg).invoke(initial_state)
315
+
316
+ from agent.graph import SYNTHESIS_MAX_TOKENS
317
+
318
+ assert SYNTHESIS_MAX_TOKENS == 64000
319
+ assert any(
320
+ kwargs.get("max_tokens") == SYNTHESIS_MAX_TOKENS
321
+ for kwargs in calls
322
+ )
323
+
324
+
325
+ def test_synthesis_truncation_produces_explicit_partial_reason(monkeypatch):
326
+ class FakeLLM:
327
+ def bind_tools(self, tools):
328
+ return self
329
+
330
+ def invoke(self, messages):
331
+ return AIMessage(content="done", tool_calls=[])
332
+
333
+ def stream(self, messages):
334
+ yield AIMessage(
335
+ content='{"ticker": "AAPL", "company_name": "Apple',
336
+ response_metadata={"stop_reason": "max_tokens"},
337
+ )
338
+
339
+ monkeypatch.setattr("analysis.textdiff.compute", lambda ticker: [])
340
+ monkeypatch.setattr("analysis.tone_drift.compute", lambda ticker: [])
341
+ monkeypatch.setattr(
342
+ "agent.company_profile.collect_profile_evidence",
343
+ lambda ticker, include_metrics=True: [],
344
+ )
345
+ monkeypatch.setattr("agent.graph.make_chat_model", lambda *args, **kwargs: FakeLLM())
346
+
347
+ cfg = RunConfig(
348
+ provider="anthropic",
349
+ model="claude-haiku-4-5-20251001",
350
+ api_key="sk-ant-test",
351
+ )
352
+ initial_state = _state([
353
+ HumanMessage(content="Generate a research brief for AAPL."),
354
+ _tool_msg("get_financial_metrics"),
355
+ _tool_msg("search_filing"),
356
+ _tool_msg("search_filing", suffix="2"),
357
+ _tool_msg("search_transcript"),
358
+ ], 0)
359
+
360
+ final = create_graph(cfg).invoke(initial_state)
361
+ brief = final["brief"]
362
+
363
+ assert brief["status"] == "PARTIAL"
364
+ assert "truncated" in brief["evidence_notes"][0]
365
+ assert "token limit" in brief["evidence_notes"][0]
366
+ assert not brief["evidence_notes"][0].startswith("Synthesis failed validation")
367
+ assert "truncated" in final["synthesis_error"]
368
+
369
+
370
  def test_partial_brief_falls_back_to_metrics_db(monkeypatch):
371
  monkeypatch.setattr(
372
  "storage.metrics_db.get_metrics",