Spaces:
Running
Running
Claude commited on
Commit ·
4b46ff9
1
Parent(s): 5d9159c
Topic validation: repair reversed bounds and drop bad topics rather than fail
Browse filesA deployed job failed entirely because the LLM returned one topic
with end (31) < start (35). The previous validator raised on the
first malformed topic, throwing away an otherwise-valid response
and forcing the user to pay for another call.
Soften validation: when end < start, swap them (the LLM clearly
intended the smaller as start) and continue. When a topic is still
zero-length or missing a title after repair, drop just that topic
and proceed with the rest. Raise LLMTopicError only when EVERY topic
is malformed — at that point the response really is unusable.
Tests cover each of the repair paths plus the still-raises-when-
nothing-usable case.
- app/pipeline/llm_extract.py +44 -19
- tests/test_topic_segmentation.py +56 -10
app/pipeline/llm_extract.py
CHANGED
|
@@ -505,42 +505,67 @@ def _validate_topics(
|
|
| 505 |
transcript_start: float,
|
| 506 |
) -> list[Topic]:
|
| 507 |
"""Coerce LLM output into a clean, sequential schedule covering the
|
| 508 |
-
full duration. Raises LLMTopicError
|
|
|
|
|
|
|
|
|
|
| 509 |
|
| 510 |
Tolerates small rounding/overlap (≤0.5s) by snapping each topic's
|
| 511 |
start to the previous topic's end. Tolerates a small gap at the
|
| 512 |
-
head/tail by extending the first/last topic.
|
| 513 |
-
backwards ranges, or gaps larger than ~10% of the video.
|
| 514 |
"""
|
| 515 |
if not raw_topics:
|
| 516 |
raise LLMTopicError("LLM returned no topics.")
|
| 517 |
-
#
|
| 518 |
-
|
| 519 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 520 |
Topic(
|
| 521 |
-
start_seconds=
|
| 522 |
-
end_seconds=
|
| 523 |
-
title=
|
| 524 |
summary=(t.summary or "").strip(),
|
| 525 |
)
|
| 526 |
-
|
| 527 |
-
|
| 528 |
-
|
| 529 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 530 |
out: list[Topic] = []
|
| 531 |
prev_end = transcript_start
|
| 532 |
for t in items:
|
| 533 |
-
if not t.title:
|
| 534 |
-
raise LLMTopicError("A topic is missing a title.")
|
| 535 |
-
if t.end_seconds <= t.start_seconds:
|
| 536 |
-
raise LLMTopicError(
|
| 537 |
-
f"Topic '{t.title}' has end ({t.end_seconds}) <= start ({t.start_seconds})."
|
| 538 |
-
)
|
| 539 |
# Snap to previous end if the model produced a small overlap or gap.
|
| 540 |
snapped_start = max(prev_end, t.start_seconds)
|
| 541 |
if snapped_start >= t.end_seconds:
|
| 542 |
# Skip degenerate topics that collapse after snapping; the
|
| 543 |
# previous topic effectively swallowed this one.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 544 |
continue
|
| 545 |
out.append(
|
| 546 |
Topic(
|
|
|
|
| 505 |
transcript_start: float,
|
| 506 |
) -> list[Topic]:
|
| 507 |
"""Coerce LLM output into a clean, sequential schedule covering the
|
| 508 |
+
full duration. Raises LLMTopicError only when the output is wholly
|
| 509 |
+
unsalvageable (empty, every topic malformed, every topic missing a
|
| 510 |
+
title); individual malformed topics are repaired when possible
|
| 511 |
+
(swapping a reversed start/end) and dropped with a warning when not.
|
| 512 |
|
| 513 |
Tolerates small rounding/overlap (≤0.5s) by snapping each topic's
|
| 514 |
start to the previous topic's end. Tolerates a small gap at the
|
| 515 |
+
head/tail by extending the first/last topic.
|
|
|
|
| 516 |
"""
|
| 517 |
if not raw_topics:
|
| 518 |
raise LLMTopicError("LLM returned no topics.")
|
| 519 |
+
# Pre-clean: drop title-less topics, repair reversed start/end pairs
|
| 520 |
+
# (the LLM occasionally swaps them when paraphrasing the timestamps),
|
| 521 |
+
# drop anything that's still degenerate.
|
| 522 |
+
cleaned: list[Topic] = []
|
| 523 |
+
for t in raw_topics:
|
| 524 |
+
title = t.title.strip()
|
| 525 |
+
if not title:
|
| 526 |
+
log.warning("Dropping topic with empty title (%.1f-%.1f)", t.start_seconds, t.end_seconds)
|
| 527 |
+
continue
|
| 528 |
+
start = float(t.start_seconds)
|
| 529 |
+
end = float(t.end_seconds)
|
| 530 |
+
if end < start:
|
| 531 |
+
log.warning(
|
| 532 |
+
"Repairing reversed topic '%s' (start=%.1f, end=%.1f) by swapping bounds",
|
| 533 |
+
title, start, end,
|
| 534 |
+
)
|
| 535 |
+
start, end = end, start
|
| 536 |
+
if end <= start:
|
| 537 |
+
log.warning(
|
| 538 |
+
"Dropping degenerate topic '%s' (start=%.1f, end=%.1f)",
|
| 539 |
+
title, start, end,
|
| 540 |
+
)
|
| 541 |
+
continue
|
| 542 |
+
cleaned.append(
|
| 543 |
Topic(
|
| 544 |
+
start_seconds=start,
|
| 545 |
+
end_seconds=end,
|
| 546 |
+
title=title,
|
| 547 |
summary=(t.summary or "").strip(),
|
| 548 |
)
|
| 549 |
+
)
|
| 550 |
+
if not cleaned:
|
| 551 |
+
raise LLMTopicError(
|
| 552 |
+
"Every topic the LLM returned was malformed (missing title, reversed, or zero-length)."
|
| 553 |
+
)
|
| 554 |
+
# Sort by start in case the model returned them slightly out of order
|
| 555 |
+
# (or a swap above changed the order).
|
| 556 |
+
items = sorted(cleaned, key=lambda t: t.start_seconds)
|
| 557 |
out: list[Topic] = []
|
| 558 |
prev_end = transcript_start
|
| 559 |
for t in items:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 560 |
# Snap to previous end if the model produced a small overlap or gap.
|
| 561 |
snapped_start = max(prev_end, t.start_seconds)
|
| 562 |
if snapped_start >= t.end_seconds:
|
| 563 |
# Skip degenerate topics that collapse after snapping; the
|
| 564 |
# previous topic effectively swallowed this one.
|
| 565 |
+
log.info(
|
| 566 |
+
"Topic '%s' was swallowed by its predecessor after snapping; dropping.",
|
| 567 |
+
t.title,
|
| 568 |
+
)
|
| 569 |
continue
|
| 570 |
out.append(
|
| 571 |
Topic(
|
tests/test_topic_segmentation.py
CHANGED
|
@@ -32,19 +32,65 @@ def test_validate_topics_rejects_empty_list():
|
|
| 32 |
_validate_topics([], total_duration=60.0, transcript_start=0.0)
|
| 33 |
|
| 34 |
|
| 35 |
-
def
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
|
| 43 |
|
| 44 |
-
def
|
| 45 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 46 |
_validate_topics(
|
| 47 |
-
[_et(10.0,
|
| 48 |
total_duration=60.0,
|
| 49 |
transcript_start=0.0,
|
| 50 |
)
|
|
|
|
| 32 |
_validate_topics([], total_duration=60.0, transcript_start=0.0)
|
| 33 |
|
| 34 |
|
| 35 |
+
def test_validate_topics_drops_missing_title_keeps_valid_neighbours():
|
| 36 |
+
"""A single missing-title topic from the LLM shouldn't tank the
|
| 37 |
+
whole job — drop it and keep the rest.
|
| 38 |
+
"""
|
| 39 |
+
out = _validate_topics(
|
| 40 |
+
[_et(0.0, 30.0, title=""), _et(30.0, 60.0, title="ok")],
|
| 41 |
+
total_duration=60.0,
|
| 42 |
+
transcript_start=0.0,
|
| 43 |
+
)
|
| 44 |
+
assert len(out) == 1
|
| 45 |
+
assert out[0].title == "ok"
|
| 46 |
|
| 47 |
|
| 48 |
+
def test_validate_topics_repairs_reversed_start_end_by_swapping():
|
| 49 |
+
"""Real failure mode from a deployed job: the LLM returned a topic
|
| 50 |
+
with end (31) before start (35). Previously we raised and failed
|
| 51 |
+
the whole job. Now we swap and keep going.
|
| 52 |
+
"""
|
| 53 |
+
out = _validate_topics(
|
| 54 |
+
[
|
| 55 |
+
_et(0.0, 30.0, "Intro"),
|
| 56 |
+
_et(35.0, 31.0, "Why climate education matters"), # reversed
|
| 57 |
+
_et(40.0, 60.0, "Outro"),
|
| 58 |
+
],
|
| 59 |
+
total_duration=60.0,
|
| 60 |
+
transcript_start=0.0,
|
| 61 |
+
)
|
| 62 |
+
titles = [t.title for t in out]
|
| 63 |
+
assert "Why climate education matters" in titles
|
| 64 |
+
repaired = next(t for t in out if t.title == "Why climate education matters")
|
| 65 |
+
assert repaired.start_seconds <= repaired.end_seconds
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def test_validate_topics_drops_zero_length_topic_keeps_neighbours():
|
| 69 |
+
"""A topic with start == end (still degenerate after repair) is
|
| 70 |
+
dropped; the rest survive.
|
| 71 |
+
"""
|
| 72 |
+
out = _validate_topics(
|
| 73 |
+
[
|
| 74 |
+
_et(0.0, 30.0, "Intro"),
|
| 75 |
+
_et(30.0, 30.0, "Zero length"),
|
| 76 |
+
_et(30.0, 60.0, "Outro"),
|
| 77 |
+
],
|
| 78 |
+
total_duration=60.0,
|
| 79 |
+
transcript_start=0.0,
|
| 80 |
+
)
|
| 81 |
+
titles = [t.title for t in out]
|
| 82 |
+
assert "Zero length" not in titles
|
| 83 |
+
assert "Intro" in titles and "Outro" in titles
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def test_validate_topics_raises_when_every_topic_is_malformed():
|
| 87 |
+
"""If literally every topic is unsalvageable, the LLM call is a
|
| 88 |
+
bust — raise so the job surfaces a real error rather than
|
| 89 |
+
silently producing an empty guide.
|
| 90 |
+
"""
|
| 91 |
+
with pytest.raises(LLMTopicError, match="malformed"):
|
| 92 |
_validate_topics(
|
| 93 |
+
[_et(10.0, 10.0, title=""), _et(20.0, 20.0, title="")],
|
| 94 |
total_duration=60.0,
|
| 95 |
transcript_start=0.0,
|
| 96 |
)
|