Claude commited on
Commit
c481e30
Β·
1 Parent(s): 0070798

feat: prev button, step instructions all tabs, noise threshold 3.2, word corrections I->T G->s

Browse files
smoke_signal/scripts/punct_corrector.py CHANGED
@@ -26,7 +26,6 @@ SMART_QUOTES = {"β€œ", "”", "β€˜", "’"}
26
 
27
  KNOWN_SUBSTITUTIONS: dict[str, str] = {
28
  "l": "!",
29
- "I": "!",
30
  "|": "!",
31
  "1": "!",
32
  ",,": "\"",
@@ -35,6 +34,23 @@ KNOWN_SUBSTITUTIONS: dict[str, str] = {
35
  "ΚΌ": "'",
36
  }
37
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
38
  MIN_PUNCT_PER_10_WORDS = 0.8
39
 
40
 
@@ -237,6 +253,15 @@ def apply_punctuation_corrections(
237
  result_chars.append(learned if learned else ch)
238
  text = "".join(result_chars)
239
 
 
 
 
 
 
 
 
 
 
240
  flags = validate_punctuation(text)
241
  words = _word_count(text)
242
  punct = _punct_count(text)
 
26
 
27
  KNOWN_SUBSTITUTIONS: dict[str, str] = {
28
  "l": "!",
 
29
  "|": "!",
30
  "1": "!",
31
  ",,": "\"",
 
34
  "ΚΌ": "'",
35
  }
36
 
37
+ # Whole-word corrections for known Tesseract errors on Gruffalo-style fonts.
38
+ # I misread as T at word start; G misread as s.
39
+ # Extend this as more errors are found in review.
40
+ WORD_CORRECTIONS: dict[str, str] = {
41
+ "Tndeed": "Indeed",
42
+ "Tll": "I'll",
43
+ "Tt": "It",
44
+ "Ti": "It",
45
+ "sood": "good",
46
+ "sreat": "great",
47
+ "sot": "got",
48
+ "sround": "ground",
49
+ "srew": "grew",
50
+ "save": "gave",
51
+ "soing": "going",
52
+ }
53
+
54
  MIN_PUNCT_PER_10_WORDS = 0.8
55
 
56
 
 
253
  result_chars.append(learned if learned else ch)
254
  text = "".join(result_chars)
255
 
256
+ # Apply whole-word corrections (Tesseract font-specific errors)
257
+ import re as _re
258
+ def _fix_word(m):
259
+ w = m.group(0)
260
+ return WORD_CORRECTIONS.get(w, w)
261
+ pattern = "|".join(_re.escape(k) for k in WORD_CORRECTIONS)
262
+ if pattern:
263
+ text = _re.sub(pattern, _fix_word, text)
264
+
265
  flags = validate_punctuation(text)
266
  words = _word_count(text)
267
  punct = _punct_count(text)
smoke_signal_tab.py CHANGED
@@ -1752,7 +1752,7 @@ def _run_tesseract_batch(
1752
  printable = sum(1 for c in txt if c.isprintable() and not c.isspace())
1753
  r_alpha = (letters / printable) if printable else 0.0
1754
  # Keep if: long enough tokens OR high confidence OR single short word (punctuation line)
1755
- if r_avg_tok >= 3.0 or r["confidence"] >= 0.80 or (len(tokens) == 1 and r["confidence"] >= 0.60):
1756
  clean_regions.append(r)
1757
  if not clean_regions:
1758
  # Everything filtered β€” page is pure illustration noise
@@ -2864,6 +2864,13 @@ def smoke_signal_tab():
2864
  <div class="ss-panel-icon">πŸ‘</div>
2865
  <div><p class="ss-panel-title">OCR Engine</p>
2866
  <p class="ss-panel-sub">Surya layout + recognition Β· Confidence scoring Β· Review queue</p></div>
 
 
 
 
 
 
 
2867
  </div>""")
2868
 
2869
  ocr_status = gr.HTML(_ocr_status_html())
@@ -2932,6 +2939,13 @@ def smoke_signal_tab():
2932
  <div class="ss-panel-icon">✎</div>
2933
  <div><p class="ss-panel-title">Review Workbench</p>
2934
  <p class="ss-panel-sub">Correct Β· Accept Β· Reject Β· Build gold training set</p></div>
 
 
 
 
 
 
 
2935
  </div>""")
2936
 
2937
  review_status = gr.HTML(_review_status_html())
@@ -2972,7 +2986,9 @@ def smoke_signal_tab():
2972
  with gr.Row():
2973
  reject_btn = gr.Button("βœ— Reject", elem_classes=["ss-btn-reject"])
2974
  quar_btn = gr.Button("βš‘ Quarantine", elem_classes=["ss-btn-quar"])
2975
- next_btn = gr.Button("Next Item β†’", elem_classes=["ss-btn-next"])
 
 
2976
 
2977
  load_review_btn = gr.Button("↻ Load Review Queue", elem_classes=["ss-btn-next"])
2978
  current_idx = gr.State(0)
@@ -3034,6 +3050,13 @@ def smoke_signal_tab():
3034
  img, raw, info, _, _ = get_review_item(new_idx)
3035
  return new_idx, img, raw, raw, info
3036
 
 
 
 
 
 
 
 
3037
  next_btn.click(_next, inputs=[current_idx],
3038
  outputs=[current_idx, review_image, raw_text_box, final_text_box, item_info])
3039
 
 
1752
  printable = sum(1 for c in txt if c.isprintable() and not c.isspace())
1753
  r_alpha = (letters / printable) if printable else 0.0
1754
  # Keep if: long enough tokens OR high confidence OR single short word (punctuation line)
1755
+ if r_avg_tok >= 3.2 or r["confidence"] >= 0.82 or (len(tokens) == 1 and r["confidence"] >= 0.65):
1756
  clean_regions.append(r)
1757
  if not clean_regions:
1758
  # Everything filtered β€” page is pure illustration noise
 
2864
  <div class="ss-panel-icon">πŸ‘</div>
2865
  <div><p class="ss-panel-title">OCR Engine</p>
2866
  <p class="ss-panel-sub">Surya layout + recognition Β· Confidence scoring Β· Review queue</p></div>
2867
+ </div>
2868
+ <div style="background:#1a2535;border:1px solid #30363d;border-radius:8px;padding:12px 18px;margin:12px 0;font-size:12px;color:#8b949e;line-height:1.8">
2869
+ <b style="color:#e6edf3">&#9654; Order:</b>
2870
+ <span style="color:#3fb950">β‘ </span> Ingest + Save Book Pages &nbsp;
2871
+ <span style="color:#3fb950">β‘‘</span> Click <b style="color:#f0883e">Run OCR</b> below &nbsp;
2872
+ <span style="color:#3fb950">β‘’</span> Wait for log to complete &nbsp;
2873
+ <span style="color:#3fb950">β‘£</span> Go to Review tab
2874
  </div>""")
2875
 
2876
  ocr_status = gr.HTML(_ocr_status_html())
 
2939
  <div class="ss-panel-icon">✎</div>
2940
  <div><p class="ss-panel-title">Review Workbench</p>
2941
  <p class="ss-panel-sub">Correct Β· Accept Β· Reject Β· Build gold training set</p></div>
2942
+ </div>
2943
+ <div style="background:#1a2535;border:1px solid #30363d;border-radius:8px;padding:12px 18px;margin:12px 0;font-size:12px;color:#8b949e;line-height:1.8">
2944
+ <b style="color:#e6edf3">&#9654; Order:</b>
2945
+ <span style="color:#3fb950">β‘ </span> Click <b style="color:#f0883e">↻ Load Review Queue</b> &nbsp;
2946
+ <span style="color:#3fb950">β‘‘</span> Review the page image and text &nbsp;
2947
+ <span style="color:#3fb950">β‘’</span> Edit text if needed Β· click <b style="color:#f0883e">βœ“ Accept</b>, <b style="color:#f0883e">✎ Save Edit</b>, or <b style="color:#f0883e">βœ— Reject</b> &nbsp;
2948
+ <span style="color:#3fb950">β‘£</span> Use <b style="color:#f0883e">← Prev</b> / <b style="color:#f0883e">Next β†’</b> to navigate without deciding
2949
  </div>""")
2950
 
2951
  review_status = gr.HTML(_review_status_html())
 
2986
  with gr.Row():
2987
  reject_btn = gr.Button("βœ— Reject", elem_classes=["ss-btn-reject"])
2988
  quar_btn = gr.Button("βš‘ Quarantine", elem_classes=["ss-btn-quar"])
2989
+ with gr.Row():
2990
+ prev_btn = gr.Button("← Prev", elem_classes=["ss-btn-next"])
2991
+ next_btn = gr.Button("Next β†’", elem_classes=["ss-btn-next"])
2992
 
2993
  load_review_btn = gr.Button("↻ Load Review Queue", elem_classes=["ss-btn-next"])
2994
  current_idx = gr.State(0)
 
3050
  img, raw, info, _, _ = get_review_item(new_idx)
3051
  return new_idx, img, raw, raw, info
3052
 
3053
+ def _prev(idx):
3054
+ new_idx = max(0, idx - 1)
3055
+ img, raw, info, _, _ = get_review_item(new_idx)
3056
+ return new_idx, img, raw, raw, info
3057
+
3058
+ prev_btn.click(_prev, inputs=[current_idx],
3059
+ outputs=[current_idx, review_image, raw_text_box, final_text_box, item_info])
3060
  next_btn.click(_next, inputs=[current_idx],
3061
  outputs=[current_idx, review_image, raw_text_box, final_text_box, item_info])
3062