Spaces:
Sleeping
Sleeping
Claude commited on
Commit Β·
c481e30
1
Parent(s): 0070798
feat: prev button, step instructions all tabs, noise threshold 3.2, word corrections I->T G->s
Browse files- smoke_signal/scripts/punct_corrector.py +26 -1
- smoke_signal_tab.py +25 -2
smoke_signal/scripts/punct_corrector.py
CHANGED
|
@@ -26,7 +26,6 @@ SMART_QUOTES = {"β", "β", "β", "β"}
|
|
| 26 |
|
| 27 |
KNOWN_SUBSTITUTIONS: dict[str, str] = {
|
| 28 |
"l": "!",
|
| 29 |
-
"I": "!",
|
| 30 |
"|": "!",
|
| 31 |
"1": "!",
|
| 32 |
",,": "\"",
|
|
@@ -35,6 +34,23 @@ KNOWN_SUBSTITUTIONS: dict[str, str] = {
|
|
| 35 |
"ΚΌ": "'",
|
| 36 |
}
|
| 37 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
MIN_PUNCT_PER_10_WORDS = 0.8
|
| 39 |
|
| 40 |
|
|
@@ -237,6 +253,15 @@ def apply_punctuation_corrections(
|
|
| 237 |
result_chars.append(learned if learned else ch)
|
| 238 |
text = "".join(result_chars)
|
| 239 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 240 |
flags = validate_punctuation(text)
|
| 241 |
words = _word_count(text)
|
| 242 |
punct = _punct_count(text)
|
|
|
|
| 26 |
|
| 27 |
KNOWN_SUBSTITUTIONS: dict[str, str] = {
|
| 28 |
"l": "!",
|
|
|
|
| 29 |
"|": "!",
|
| 30 |
"1": "!",
|
| 31 |
",,": "\"",
|
|
|
|
| 34 |
"ΚΌ": "'",
|
| 35 |
}
|
| 36 |
|
| 37 |
+
# Whole-word corrections for known Tesseract errors on Gruffalo-style fonts.
|
| 38 |
+
# I misread as T at word start; G misread as s.
|
| 39 |
+
# Extend this as more errors are found in review.
|
| 40 |
+
WORD_CORRECTIONS: dict[str, str] = {
|
| 41 |
+
"Tndeed": "Indeed",
|
| 42 |
+
"Tll": "I'll",
|
| 43 |
+
"Tt": "It",
|
| 44 |
+
"Ti": "It",
|
| 45 |
+
"sood": "good",
|
| 46 |
+
"sreat": "great",
|
| 47 |
+
"sot": "got",
|
| 48 |
+
"sround": "ground",
|
| 49 |
+
"srew": "grew",
|
| 50 |
+
"save": "gave",
|
| 51 |
+
"soing": "going",
|
| 52 |
+
}
|
| 53 |
+
|
| 54 |
MIN_PUNCT_PER_10_WORDS = 0.8
|
| 55 |
|
| 56 |
|
|
|
|
| 253 |
result_chars.append(learned if learned else ch)
|
| 254 |
text = "".join(result_chars)
|
| 255 |
|
| 256 |
+
# Apply whole-word corrections (Tesseract font-specific errors)
|
| 257 |
+
import re as _re
|
| 258 |
+
def _fix_word(m):
|
| 259 |
+
w = m.group(0)
|
| 260 |
+
return WORD_CORRECTIONS.get(w, w)
|
| 261 |
+
pattern = "|".join(_re.escape(k) for k in WORD_CORRECTIONS)
|
| 262 |
+
if pattern:
|
| 263 |
+
text = _re.sub(pattern, _fix_word, text)
|
| 264 |
+
|
| 265 |
flags = validate_punctuation(text)
|
| 266 |
words = _word_count(text)
|
| 267 |
punct = _punct_count(text)
|
smoke_signal_tab.py
CHANGED
|
@@ -1752,7 +1752,7 @@ def _run_tesseract_batch(
|
|
| 1752 |
printable = sum(1 for c in txt if c.isprintable() and not c.isspace())
|
| 1753 |
r_alpha = (letters / printable) if printable else 0.0
|
| 1754 |
# Keep if: long enough tokens OR high confidence OR single short word (punctuation line)
|
| 1755 |
-
if r_avg_tok >= 3.
|
| 1756 |
clean_regions.append(r)
|
| 1757 |
if not clean_regions:
|
| 1758 |
# Everything filtered β page is pure illustration noise
|
|
@@ -2864,6 +2864,13 @@ def smoke_signal_tab():
|
|
| 2864 |
<div class="ss-panel-icon">π</div>
|
| 2865 |
<div><p class="ss-panel-title">OCR Engine</p>
|
| 2866 |
<p class="ss-panel-sub">Surya layout + recognition Β· Confidence scoring Β· Review queue</p></div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2867 |
</div>""")
|
| 2868 |
|
| 2869 |
ocr_status = gr.HTML(_ocr_status_html())
|
|
@@ -2932,6 +2939,13 @@ def smoke_signal_tab():
|
|
| 2932 |
<div class="ss-panel-icon">β</div>
|
| 2933 |
<div><p class="ss-panel-title">Review Workbench</p>
|
| 2934 |
<p class="ss-panel-sub">Correct Β· Accept Β· Reject Β· Build gold training set</p></div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2935 |
</div>""")
|
| 2936 |
|
| 2937 |
review_status = gr.HTML(_review_status_html())
|
|
@@ -2972,7 +2986,9 @@ def smoke_signal_tab():
|
|
| 2972 |
with gr.Row():
|
| 2973 |
reject_btn = gr.Button("β Reject", elem_classes=["ss-btn-reject"])
|
| 2974 |
quar_btn = gr.Button("β Quarantine", elem_classes=["ss-btn-quar"])
|
| 2975 |
-
|
|
|
|
|
|
|
| 2976 |
|
| 2977 |
load_review_btn = gr.Button("β» Load Review Queue", elem_classes=["ss-btn-next"])
|
| 2978 |
current_idx = gr.State(0)
|
|
@@ -3034,6 +3050,13 @@ def smoke_signal_tab():
|
|
| 3034 |
img, raw, info, _, _ = get_review_item(new_idx)
|
| 3035 |
return new_idx, img, raw, raw, info
|
| 3036 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3037 |
next_btn.click(_next, inputs=[current_idx],
|
| 3038 |
outputs=[current_idx, review_image, raw_text_box, final_text_box, item_info])
|
| 3039 |
|
|
|
|
| 1752 |
printable = sum(1 for c in txt if c.isprintable() and not c.isspace())
|
| 1753 |
r_alpha = (letters / printable) if printable else 0.0
|
| 1754 |
# Keep if: long enough tokens OR high confidence OR single short word (punctuation line)
|
| 1755 |
+
if r_avg_tok >= 3.2 or r["confidence"] >= 0.82 or (len(tokens) == 1 and r["confidence"] >= 0.65):
|
| 1756 |
clean_regions.append(r)
|
| 1757 |
if not clean_regions:
|
| 1758 |
# Everything filtered β page is pure illustration noise
|
|
|
|
| 2864 |
<div class="ss-panel-icon">π</div>
|
| 2865 |
<div><p class="ss-panel-title">OCR Engine</p>
|
| 2866 |
<p class="ss-panel-sub">Surya layout + recognition Β· Confidence scoring Β· Review queue</p></div>
|
| 2867 |
+
</div>
|
| 2868 |
+
<div style="background:#1a2535;border:1px solid #30363d;border-radius:8px;padding:12px 18px;margin:12px 0;font-size:12px;color:#8b949e;line-height:1.8">
|
| 2869 |
+
<b style="color:#e6edf3">▶ Order:</b>
|
| 2870 |
+
<span style="color:#3fb950">β </span> Ingest + Save Book Pages
|
| 2871 |
+
<span style="color:#3fb950">β‘</span> Click <b style="color:#f0883e">Run OCR</b> below
|
| 2872 |
+
<span style="color:#3fb950">β’</span> Wait for log to complete
|
| 2873 |
+
<span style="color:#3fb950">β£</span> Go to Review tab
|
| 2874 |
</div>""")
|
| 2875 |
|
| 2876 |
ocr_status = gr.HTML(_ocr_status_html())
|
|
|
|
| 2939 |
<div class="ss-panel-icon">β</div>
|
| 2940 |
<div><p class="ss-panel-title">Review Workbench</p>
|
| 2941 |
<p class="ss-panel-sub">Correct Β· Accept Β· Reject Β· Build gold training set</p></div>
|
| 2942 |
+
</div>
|
| 2943 |
+
<div style="background:#1a2535;border:1px solid #30363d;border-radius:8px;padding:12px 18px;margin:12px 0;font-size:12px;color:#8b949e;line-height:1.8">
|
| 2944 |
+
<b style="color:#e6edf3">▶ Order:</b>
|
| 2945 |
+
<span style="color:#3fb950">β </span> Click <b style="color:#f0883e">β» Load Review Queue</b>
|
| 2946 |
+
<span style="color:#3fb950">β‘</span> Review the page image and text
|
| 2947 |
+
<span style="color:#3fb950">β’</span> Edit text if needed Β· click <b style="color:#f0883e">β Accept</b>, <b style="color:#f0883e">β Save Edit</b>, or <b style="color:#f0883e">β Reject</b>
|
| 2948 |
+
<span style="color:#3fb950">β£</span> Use <b style="color:#f0883e">β Prev</b> / <b style="color:#f0883e">Next β</b> to navigate without deciding
|
| 2949 |
</div>""")
|
| 2950 |
|
| 2951 |
review_status = gr.HTML(_review_status_html())
|
|
|
|
| 2986 |
with gr.Row():
|
| 2987 |
reject_btn = gr.Button("β Reject", elem_classes=["ss-btn-reject"])
|
| 2988 |
quar_btn = gr.Button("β Quarantine", elem_classes=["ss-btn-quar"])
|
| 2989 |
+
with gr.Row():
|
| 2990 |
+
prev_btn = gr.Button("β Prev", elem_classes=["ss-btn-next"])
|
| 2991 |
+
next_btn = gr.Button("Next β", elem_classes=["ss-btn-next"])
|
| 2992 |
|
| 2993 |
load_review_btn = gr.Button("β» Load Review Queue", elem_classes=["ss-btn-next"])
|
| 2994 |
current_idx = gr.State(0)
|
|
|
|
| 3050 |
img, raw, info, _, _ = get_review_item(new_idx)
|
| 3051 |
return new_idx, img, raw, raw, info
|
| 3052 |
|
| 3053 |
+
def _prev(idx):
|
| 3054 |
+
new_idx = max(0, idx - 1)
|
| 3055 |
+
img, raw, info, _, _ = get_review_item(new_idx)
|
| 3056 |
+
return new_idx, img, raw, raw, info
|
| 3057 |
+
|
| 3058 |
+
prev_btn.click(_prev, inputs=[current_idx],
|
| 3059 |
+
outputs=[current_idx, review_image, raw_text_box, final_text_box, item_info])
|
| 3060 |
next_btn.click(_next, inputs=[current_idx],
|
| 3061 |
outputs=[current_idx, review_image, raw_text_box, final_text_box, item_info])
|
| 3062 |
|