"""Tests for the inverted-pass near-duplicate filter in ocr_frame. Tesseract gets run twice in ocr_frame — once on the normal binarization (for dark-on-light body text) and once on the inverted binarization (for white-on-coloured callout text). The inverted pass on a normal slide often produces garbled near-duplicates of the normal-pass lines because inverting black-on-white produces noisy intermediate values; without a fuzzy filter the rendered guide showed both versions stacked together. """ from app.pipeline.ocr import _is_near_dup, _norm_for_match def test_garbled_inverted_pass_line_is_filtered_against_clean_normal_pass(): # Real-world example reported by a user: clean normal pass, garbled # inverted pass of the same headline. clean = "How do we increase heat pump installation rates?" garbled = "ow co we iharease lice? Gunip installation cakes?" assert _is_near_dup(garbled, [clean]) is True def test_distinct_callout_text_is_kept(): # The whole point of the inverted pass is to recover white-on-colour # callout text that the normal pass turns into noise. A genuinely # distinct callout line must NOT trigger the near-dup filter. body = "Heat pumps reduce household emissions by up to 60 percent." callout = "Key takeaway: incentives drive adoption." assert _is_near_dup(callout, [body]) is False def test_short_unrelated_line_is_kept(): # Short labels (axis labels, captions) often appear only in one pass. assert _is_near_dup("Figure 3", ["Heat pump installation rates by year"]) is False def test_punctuation_and_case_dont_block_match(): # Normalization strips punctuation and case so OCR drift on those # doesn't defeat the fuzzy comparison. a = "Heat-pump installation rates" b = "HEAT PUMP INSTALLATION RATES." assert _is_near_dup(b, [a]) is True def test_empty_candidate_or_existing_is_safe(): assert _is_near_dup("", ["something"]) is False assert _is_near_dup("something", []) is False assert _is_near_dup("something", [""]) is False def test_norm_for_match_strips_non_alnum(): assert _norm_for_match("Hello, World! 123") == "helloworld123" assert _norm_for_match(" ") == "" def test_has_real_text_keeps_short_real_labels(): """Short legitimate labels — "Lab", year-only slides, "Slide 3" — must survive the noise filter even if they're well under the typical sentence length we'd see on a bullet slide. (Punctuation- split initialisms like "Q&A" don't qualify; that's a known corner-case trade-off against being too permissive with noise.) """ from app.pipeline.ocr import _has_real_text assert _has_real_text("Lab") assert _has_real_text("1979") assert _has_real_text("Slide 3") assert _has_real_text("Module 3 Overview") def test_has_real_text_rejects_tesseract_noise(): """Tesseract on a textless slide (photo, diagram, talking-head) emits stray punctuation and single letters. None of that should pass — we want ocr_frame to return "" for those frames so the rendered guide hides the "On-screen text" section. """ from app.pipeline.ocr import _has_real_text assert not _has_real_text("") assert not _has_real_text(" \n\n") assert not _has_real_text("/| _ . ,") assert not _has_real_text("a b c") # all 1-char runs assert not _has_real_text("|a|b|") def _tesseract_data(words: list[tuple[str, int, int]]): """Helper: build a pytesseract.image_to_data-shaped dict from a list of (word, conf, line_num) tuples. Block/par are flat 0; line groups words into the same OCR-line as the caller specifies. """ n = len(words) return { "text": [w for w, _, _ in words], "conf": [c for _, c, _ in words], "block_num": [0] * n, "par_num": [0] * n, "line_num": [l for _, _, l in words], } def test_ocr_frame_returns_empty_when_frame_has_no_real_text(monkeypatch, tmp_path): """End-to-end: when every word Tesseract emits is low-confidence noise, ocr_frame must return "" rather than a noise-filled string. The rendered guide's `{% if seg.ocr_text %}` check then hides the panel entirely. """ from PIL import Image as _Image from app.pipeline import ocr as ocr_mod normal = _Image.new("L", (10, 10), color=255) inverted = _Image.new("L", (10, 10), color=0) monkeypatch.setattr(ocr_mod, "_preprocess_variants", lambda _p: [normal, inverted]) # All words below the 60-confidence floor — pure noise. noise = _tesseract_data([("ee", 20, 0), ("Re", 25, 1), ("SS", 18, 2)]) monkeypatch.setattr( ocr_mod.pytesseract, "image_to_data", lambda *a, **kw: noise ) assert ocr_mod.ocr_frame(tmp_path / "noise.jpg") == "" assert ocr_mod.ocr_frame(tmp_path / "noise.jpg", skip_inverted=True) == "" def test_ocr_frame_drops_lone_high_confidence_hallucination_surrounded_by_noise(monkeypatch, tmp_path): """The exact failure from the heat-pump AC unit screenshot: Tesseract hallucinated "Soweer" at high confidence but the surrounding lines were all sub-60 noise. The confidence filter must drop the noise lines, leaving "Soweer" alone — and since "Soweer" still has a 3+ char alphanumeric run, it survives the final sanity check. Better than the previous output which was "Soweer" PLUS a dozen lines of "ee", "Re Re", "SS ESS" gibberish. """ from PIL import Image as _Image from app.pipeline import ocr as ocr_mod normal = _Image.new("L", (10, 10), color=255) inverted = _Image.new("L", (10, 10), color=0) monkeypatch.setattr(ocr_mod, "_preprocess_variants", lambda _p: [normal, inverted]) monkeypatch.setattr( ocr_mod.pytesseract, "image_to_data", lambda *a, **kw: _tesseract_data( [ ("Soweer", 75, 0), # high-confidence hallucination — kept ("ee", 28, 1), # noise — dropped ("es", 31, 2), # noise — dropped ("Re", 22, 2), # noise — dropped ] ), ) out = ocr_mod.ocr_frame(tmp_path / "ac.jpg") assert "Soweer" in out # The low-confidence noise must NOT appear as standalone words/lines. # (Substring match isn't valid here — "ee" is inside "Soweer".) lines = [l.strip() for l in out.splitlines() if l.strip()] for noise in ("ee", "es", "Re", "es Re", "Re Re"): assert noise not in lines, f"noise line '{noise}' should have been filtered" def test_ocr_frame_keeps_real_slide_text(monkeypatch, tmp_path): """Sanity guard: the confidence filter must not eat high-confidence real OCR output. """ from PIL import Image as _Image from app.pipeline import ocr as ocr_mod normal = _Image.new("L", (10, 10), color=255) inverted = _Image.new("L", (10, 10), color=0) monkeypatch.setattr(ocr_mod, "_preprocess_variants", lambda _p: [normal, inverted]) monkeypatch.setattr( ocr_mod.pytesseract, "image_to_data", lambda *a, **kw: _tesseract_data( [ ("Learning", 88, 0), ("outcomes", 92, 0), ("for", 84, 0), ("module", 90, 0), ("3", 85, 0), ] ), ) out = ocr_mod.ocr_frame(tmp_path / "slide.jpg") assert "Learning outcomes" in out assert "module 3" in out def test_skip_inverted_only_runs_normal_pass(monkeypatch, tmp_path): """When skip_inverted=True, ocr_frame must not invoke Tesseract on the inverted variant — that's the whole point of the flag (halves OCR time and rules out inverted-pass garbling). """ from PIL import Image as _Image from app.pipeline import ocr as ocr_mod normal = _Image.new("L", (10, 10), color=255) inverted = _Image.new("L", (10, 10), color=0) monkeypatch.setattr(ocr_mod, "_preprocess_variants", lambda _p: [normal, inverted]) calls: list[int] = [] def fake_image_to_data(img, lang="eng", output_type=None): calls.append(id(img)) return _tesseract_data([("Body", 90, 0), ("line", 88, 0), ("one", 85, 0)]) monkeypatch.setattr(ocr_mod.pytesseract, "image_to_data", fake_image_to_data) text_skip = ocr_mod.ocr_frame(tmp_path / "fake.jpg", skip_inverted=True) assert len(calls) == 1, "skip_inverted must not invoke the inverted pass" assert "Body line one" in text_skip calls.clear() text_full = ocr_mod.ocr_frame(tmp_path / "fake.jpg", skip_inverted=False) assert len(calls) == 2, "default behaviour must run both passes" # With identical text from both variants, the dedupe collapses them. assert "Body line one" in text_full