fix: use token length as primary noise discriminator — filters td/wht/LN illustration noise

smoke_signal_tab.py

@@ -1738,19 +1738,31 @@ def _run_tesseract_batch(
                     "method": "tesseract-noise-filtered",
                     "tesseract_lang": lang_hint or "eng",
                 }
-            # 3. Filter individual regions that look like illustration noise
-            # Keep only regions where alpha ratio >= 0.60 and token length >= 2
+            # 3. Filter individual regions that look like illustration noise.
+            # Key discriminator: real picture-book words avg 3.5+ chars.
+            # Illustration noise (td, wht, LN, WA) avg 2.5 chars.
             clean_regions = []
             for r in regions:
                 txt = r["text"]
+                tokens = [t for t in txt.split() if t and any(c.isalpha() for c in t)]
+                if not tokens:
+                    continue
+                r_avg_tok = sum(len(t) for t in tokens) / len(tokens)
                 letters = sum(1 for c in txt if c.isalpha())
                 printable = sum(1 for c in txt if c.isprintable() and not c.isspace())
                 r_alpha = (letters / printable) if printable else 0.0
-                tokens = [t for t in txt.split() if t]
-                r_avg_tok = (sum(len(t) for t in tokens) / len(tokens)) if tokens else 0.0
-                if r_alpha >= 0.55 or r["confidence"] >= 0.75:
+                # Keep if: long enough tokens OR high confidence OR single short word (punctuation line)
+                if r_avg_tok >= 3.0 or r["confidence"] >= 0.80 or (len(tokens) == 1 and r["confidence"] >= 0.60):
                     clean_regions.append(r)
-            if clean_regions and len(clean_regions) < len(regions):
+            if not clean_regions:
+                # Everything filtered — page is pure illustration noise
+                return {
+                    "regions": [],
+                    "confidence": 0.0,
+                    "method": "tesseract-noise-filtered",
+                    "tesseract_lang": lang_hint or "eng",
+                }
+            if len(clean_regions) < len(regions):
                 # Recalculate confidence without noise regions
                 cw = sum(r["confidence"] * r["word_count"] for r in clean_regions)
                 wc = sum(r["word_count"] for r in clean_regions)