Claude commited on
Commit
4b46ff9
·
1 Parent(s): 5d9159c

Topic validation: repair reversed bounds and drop bad topics rather than fail

Browse files

A deployed job failed entirely because the LLM returned one topic
with end (31) < start (35). The previous validator raised on the
first malformed topic, throwing away an otherwise-valid response
and forcing the user to pay for another call.

Soften validation: when end < start, swap them (the LLM clearly
intended the smaller as start) and continue. When a topic is still
zero-length or missing a title after repair, drop just that topic
and proceed with the rest. Raise LLMTopicError only when EVERY topic
is malformed — at that point the response really is unusable.

Tests cover each of the repair paths plus the still-raises-when-
nothing-usable case.

app/pipeline/llm_extract.py CHANGED
@@ -505,42 +505,67 @@ def _validate_topics(
505
  transcript_start: float,
506
  ) -> list[Topic]:
507
  """Coerce LLM output into a clean, sequential schedule covering the
508
- full duration. Raises LLMTopicError if the output is unsalvageable.
 
 
 
509
 
510
  Tolerates small rounding/overlap (≤0.5s) by snapping each topic's
511
  start to the previous topic's end. Tolerates a small gap at the
512
- head/tail by extending the first/last topic. Rejects empty output,
513
- backwards ranges, or gaps larger than ~10% of the video.
514
  """
515
  if not raw_topics:
516
  raise LLMTopicError("LLM returned no topics.")
517
- # Sort by start in case the model returned them slightly out of order.
518
- items = sorted(
519
- [
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
520
  Topic(
521
- start_seconds=float(t.start_seconds),
522
- end_seconds=float(t.end_seconds),
523
- title=t.title.strip(),
524
  summary=(t.summary or "").strip(),
525
  )
526
- for t in raw_topics
527
- ],
528
- key=lambda t: t.start_seconds,
529
- )
 
 
 
 
530
  out: list[Topic] = []
531
  prev_end = transcript_start
532
  for t in items:
533
- if not t.title:
534
- raise LLMTopicError("A topic is missing a title.")
535
- if t.end_seconds <= t.start_seconds:
536
- raise LLMTopicError(
537
- f"Topic '{t.title}' has end ({t.end_seconds}) <= start ({t.start_seconds})."
538
- )
539
  # Snap to previous end if the model produced a small overlap or gap.
540
  snapped_start = max(prev_end, t.start_seconds)
541
  if snapped_start >= t.end_seconds:
542
  # Skip degenerate topics that collapse after snapping; the
543
  # previous topic effectively swallowed this one.
 
 
 
 
544
  continue
545
  out.append(
546
  Topic(
 
505
  transcript_start: float,
506
  ) -> list[Topic]:
507
  """Coerce LLM output into a clean, sequential schedule covering the
508
+ full duration. Raises LLMTopicError only when the output is wholly
509
+ unsalvageable (empty, every topic malformed, every topic missing a
510
+ title); individual malformed topics are repaired when possible
511
+ (swapping a reversed start/end) and dropped with a warning when not.
512
 
513
  Tolerates small rounding/overlap (≤0.5s) by snapping each topic's
514
  start to the previous topic's end. Tolerates a small gap at the
515
+ head/tail by extending the first/last topic.
 
516
  """
517
  if not raw_topics:
518
  raise LLMTopicError("LLM returned no topics.")
519
+ # Pre-clean: drop title-less topics, repair reversed start/end pairs
520
+ # (the LLM occasionally swaps them when paraphrasing the timestamps),
521
+ # drop anything that's still degenerate.
522
+ cleaned: list[Topic] = []
523
+ for t in raw_topics:
524
+ title = t.title.strip()
525
+ if not title:
526
+ log.warning("Dropping topic with empty title (%.1f-%.1f)", t.start_seconds, t.end_seconds)
527
+ continue
528
+ start = float(t.start_seconds)
529
+ end = float(t.end_seconds)
530
+ if end < start:
531
+ log.warning(
532
+ "Repairing reversed topic '%s' (start=%.1f, end=%.1f) by swapping bounds",
533
+ title, start, end,
534
+ )
535
+ start, end = end, start
536
+ if end <= start:
537
+ log.warning(
538
+ "Dropping degenerate topic '%s' (start=%.1f, end=%.1f)",
539
+ title, start, end,
540
+ )
541
+ continue
542
+ cleaned.append(
543
  Topic(
544
+ start_seconds=start,
545
+ end_seconds=end,
546
+ title=title,
547
  summary=(t.summary or "").strip(),
548
  )
549
+ )
550
+ if not cleaned:
551
+ raise LLMTopicError(
552
+ "Every topic the LLM returned was malformed (missing title, reversed, or zero-length)."
553
+ )
554
+ # Sort by start in case the model returned them slightly out of order
555
+ # (or a swap above changed the order).
556
+ items = sorted(cleaned, key=lambda t: t.start_seconds)
557
  out: list[Topic] = []
558
  prev_end = transcript_start
559
  for t in items:
 
 
 
 
 
 
560
  # Snap to previous end if the model produced a small overlap or gap.
561
  snapped_start = max(prev_end, t.start_seconds)
562
  if snapped_start >= t.end_seconds:
563
  # Skip degenerate topics that collapse after snapping; the
564
  # previous topic effectively swallowed this one.
565
+ log.info(
566
+ "Topic '%s' was swallowed by its predecessor after snapping; dropping.",
567
+ t.title,
568
+ )
569
  continue
570
  out.append(
571
  Topic(
tests/test_topic_segmentation.py CHANGED
@@ -32,19 +32,65 @@ def test_validate_topics_rejects_empty_list():
32
  _validate_topics([], total_duration=60.0, transcript_start=0.0)
33
 
34
 
35
- def test_validate_topics_rejects_missing_title():
36
- with pytest.raises(LLMTopicError, match="missing a title"):
37
- _validate_topics(
38
- [_et(0.0, 30.0, title=""), _et(30.0, 60.0, title="ok")],
39
- total_duration=60.0,
40
- transcript_start=0.0,
41
- )
 
 
 
 
42
 
43
 
44
- def test_validate_topics_rejects_backwards_range():
45
- with pytest.raises(LLMTopicError, match="end .* <= start"):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
46
  _validate_topics(
47
- [_et(10.0, 5.0)],
48
  total_duration=60.0,
49
  transcript_start=0.0,
50
  )
 
32
  _validate_topics([], total_duration=60.0, transcript_start=0.0)
33
 
34
 
35
+ def test_validate_topics_drops_missing_title_keeps_valid_neighbours():
36
+ """A single missing-title topic from the LLM shouldn't tank the
37
+ whole job — drop it and keep the rest.
38
+ """
39
+ out = _validate_topics(
40
+ [_et(0.0, 30.0, title=""), _et(30.0, 60.0, title="ok")],
41
+ total_duration=60.0,
42
+ transcript_start=0.0,
43
+ )
44
+ assert len(out) == 1
45
+ assert out[0].title == "ok"
46
 
47
 
48
+ def test_validate_topics_repairs_reversed_start_end_by_swapping():
49
+ """Real failure mode from a deployed job: the LLM returned a topic
50
+ with end (31) before start (35). Previously we raised and failed
51
+ the whole job. Now we swap and keep going.
52
+ """
53
+ out = _validate_topics(
54
+ [
55
+ _et(0.0, 30.0, "Intro"),
56
+ _et(35.0, 31.0, "Why climate education matters"), # reversed
57
+ _et(40.0, 60.0, "Outro"),
58
+ ],
59
+ total_duration=60.0,
60
+ transcript_start=0.0,
61
+ )
62
+ titles = [t.title for t in out]
63
+ assert "Why climate education matters" in titles
64
+ repaired = next(t for t in out if t.title == "Why climate education matters")
65
+ assert repaired.start_seconds <= repaired.end_seconds
66
+
67
+
68
+ def test_validate_topics_drops_zero_length_topic_keeps_neighbours():
69
+ """A topic with start == end (still degenerate after repair) is
70
+ dropped; the rest survive.
71
+ """
72
+ out = _validate_topics(
73
+ [
74
+ _et(0.0, 30.0, "Intro"),
75
+ _et(30.0, 30.0, "Zero length"),
76
+ _et(30.0, 60.0, "Outro"),
77
+ ],
78
+ total_duration=60.0,
79
+ transcript_start=0.0,
80
+ )
81
+ titles = [t.title for t in out]
82
+ assert "Zero length" not in titles
83
+ assert "Intro" in titles and "Outro" in titles
84
+
85
+
86
+ def test_validate_topics_raises_when_every_topic_is_malformed():
87
+ """If literally every topic is unsalvageable, the LLM call is a
88
+ bust — raise so the job surfaces a real error rather than
89
+ silently producing an empty guide.
90
+ """
91
+ with pytest.raises(LLMTopicError, match="malformed"):
92
  _validate_topics(
93
+ [_et(10.0, 10.0, title=""), _et(20.0, 20.0, title="")],
94
  total_duration=60.0,
95
  transcript_start=0.0,
96
  )