Spaces:
Sleeping
Sleeping
Claude commited on
Commit Β·
4233f08
1
Parent(s): c481e30
fix: ocr_done state triggers review load after OCR completes; add Gruffalo word corrections
Browse files- smoke_signal/scripts/punct_corrector.py +7 -0
- smoke_signal_tab.py +10 -12
smoke_signal/scripts/punct_corrector.py
CHANGED
|
@@ -38,10 +38,16 @@ KNOWN_SUBSTITUTIONS: dict[str, str] = {
|
|
| 38 |
# I misread as T at word start; G misread as s.
|
| 39 |
# Extend this as more errors are found in review.
|
| 40 |
WORD_CORRECTIONS: dict[str, str] = {
|
|
|
|
| 41 |
"Tndeed": "Indeed",
|
| 42 |
"Tll": "I'll",
|
| 43 |
"Tt": "It",
|
| 44 |
"Ti": "It",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
"sood": "good",
|
| 46 |
"sreat": "great",
|
| 47 |
"sot": "got",
|
|
@@ -49,6 +55,7 @@ WORD_CORRECTIONS: dict[str, str] = {
|
|
| 49 |
"srew": "grew",
|
| 50 |
"save": "gave",
|
| 51 |
"soing": "going",
|
|
|
|
| 52 |
}
|
| 53 |
|
| 54 |
MIN_PUNCT_PER_10_WORDS = 0.8
|
|
|
|
| 38 |
# I misread as T at word start; G misread as s.
|
| 39 |
# Extend this as more errors are found in review.
|
| 40 |
WORD_CORRECTIONS: dict[str, str] = {
|
| 41 |
+
# I misread as T at word start
|
| 42 |
"Tndeed": "Indeed",
|
| 43 |
"Tll": "I'll",
|
| 44 |
"Tt": "It",
|
| 45 |
"Ti": "It",
|
| 46 |
+
"Tn": "In",
|
| 47 |
+
"Ts": "Is",
|
| 48 |
+
# G misread as s at word start
|
| 49 |
+
"sruffalo": "Gruffalo",
|
| 50 |
+
"sruffalo's": "Gruffalo's",
|
| 51 |
"sood": "good",
|
| 52 |
"sreat": "great",
|
| 53 |
"sot": "got",
|
|
|
|
| 55 |
"srew": "grew",
|
| 56 |
"save": "gave",
|
| 57 |
"soing": "going",
|
| 58 |
+
"so": "Go", # only catches capitalised β won't corrupt "so" mid sentence
|
| 59 |
}
|
| 60 |
|
| 61 |
MIN_PUNCT_PER_10_WORDS = 0.8
|
smoke_signal_tab.py
CHANGED
|
@@ -1946,7 +1946,7 @@ def run_ocr(page_selection: str = "", page_exclusion: str = "", replace_book_que
|
|
| 1946 |
)
|
| 1947 |
)
|
| 1948 |
|
| 1949 |
-
yield _ocr_status_html(), "\n".join(log) # stream: models loaded
|
| 1950 |
punct_mod = _load_punct_module()
|
| 1951 |
if punct_mod is None:
|
| 1952 |
log.append(log_line(f"β Punctuation corrector unavailable ({_PUNCT_MODULE_ERROR or 'unknown'})"))
|
|
@@ -1973,7 +1973,7 @@ def run_ocr(page_selection: str = "", page_exclusion: str = "", replace_book_que
|
|
| 1973 |
for _, row in eligible.iterrows():
|
| 1974 |
book_id = row["book_id"]
|
| 1975 |
log.append(log_line(f"βΆ {book_id}: starting OCR"))
|
| 1976 |
-
yield _ocr_status_html(), "\n".join(log) # stream: book start
|
| 1977 |
profile_path = PROFILES_DIR / f"{book_id}_page_profile.json"
|
| 1978 |
if not profile_path.exists():
|
| 1979 |
log.append(log_line(f"β {book_id}: no profile"))
|
|
@@ -2239,10 +2239,10 @@ def run_ocr(page_selection: str = "", page_exclusion: str = "", replace_book_que
|
|
| 2239 |
# Stream progress after each batch
|
| 2240 |
done_pages = min(start + batch_size, len(ocr_targets))
|
| 2241 |
log.append(log_line(f" β {book_id}: batch {start//batch_size + 1} complete ({done_pages}/{len(ocr_targets)} pages)"))
|
| 2242 |
-
yield _ocr_status_html(), "\n".join(log)
|
| 2243 |
|
| 2244 |
log.append(log_line(f" β {book_id}: batch OCR complete β building review queue"))
|
| 2245 |
-
yield _ocr_status_html(), "\n".join(log) # stream: batch OCR done
|
| 2246 |
# Second pass: build page-level OCR output and review queue.
|
| 2247 |
for page_data in profile.get("pages", []):
|
| 2248 |
page_num = page_data["page_number"]
|
|
@@ -2371,7 +2371,7 @@ def run_ocr(page_selection: str = "", page_exclusion: str = "", replace_book_que
|
|
| 2371 |
df.loc[df["book_id"] == book_id, "status"] = "ocred"
|
| 2372 |
total_review = len(review_pages) + len(quarantine_pages)
|
| 2373 |
log.append(log_line(f"β {book_id}: {len(ocr_pages)} selected pp β review queue: {total_review} Β· queue write complete"))
|
| 2374 |
-
yield _ocr_status_html(), "\n".join(log) # stream: book done
|
| 2375 |
|
| 2376 |
qdf_new = pd.DataFrame(queue_rows)
|
| 2377 |
if QUEUE_CSV.exists():
|
|
@@ -2395,7 +2395,7 @@ def run_ocr(page_selection: str = "", page_exclusion: str = "", replace_book_que
|
|
| 2395 |
QUEUE_CSV.unlink()
|
| 2396 |
|
| 2397 |
save_manifest_df(df)
|
| 2398 |
-
yield _ocr_status_html(), "\n".join(log) # stream: final
|
| 2399 |
|
| 2400 |
|
| 2401 |
def _ocr_status_html() -> str:
|
|
@@ -2874,6 +2874,7 @@ def smoke_signal_tab():
|
|
| 2874 |
</div>""")
|
| 2875 |
|
| 2876 |
ocr_status = gr.HTML(_ocr_status_html())
|
|
|
|
| 2877 |
|
| 2878 |
gr.HTML("""<div class="ss-card">
|
| 2879 |
<div class="ss-card-title">Self-Improvement Loop</div>
|
|
@@ -2911,7 +2912,7 @@ def smoke_signal_tab():
|
|
| 2911 |
ocr_run_event = ocr_btn.click(
|
| 2912 |
run_ocr,
|
| 2913 |
inputs=[ocr_page_selection, ocr_page_exclusion, ocr_replace_queue],
|
| 2914 |
-
outputs=[ocr_status, ocr_log],
|
| 2915 |
show_progress="minimal",
|
| 2916 |
trigger_mode="once",
|
| 2917 |
)
|
|
@@ -3035,11 +3036,8 @@ def smoke_signal_tab():
|
|
| 3035 |
outputs=review_load_outputs
|
| 3036 |
)
|
| 3037 |
review_tab.select(load_review, outputs=review_load_outputs)
|
| 3038 |
-
|
| 3039 |
-
|
| 3040 |
-
outputs=review_load_outputs,
|
| 3041 |
-
show_progress="hidden",
|
| 3042 |
-
)
|
| 3043 |
accept_btn.click(do_accept, inputs=[current_idx, final_text_box, reviewer_name, reason_code], outputs=action_outputs)
|
| 3044 |
edit_btn.click(do_accept, inputs=[current_idx, final_text_box, reviewer_name, reason_code], outputs=action_outputs)
|
| 3045 |
reject_btn.click(do_reject, inputs=[current_idx, final_text_box, reviewer_name, reason_code], outputs=action_outputs)
|
|
|
|
| 1946 |
)
|
| 1947 |
)
|
| 1948 |
|
| 1949 |
+
yield _ocr_status_html(), "\n".join(log), 0 # stream: models loaded
|
| 1950 |
punct_mod = _load_punct_module()
|
| 1951 |
if punct_mod is None:
|
| 1952 |
log.append(log_line(f"β Punctuation corrector unavailable ({_PUNCT_MODULE_ERROR or 'unknown'})"))
|
|
|
|
| 1973 |
for _, row in eligible.iterrows():
|
| 1974 |
book_id = row["book_id"]
|
| 1975 |
log.append(log_line(f"βΆ {book_id}: starting OCR"))
|
| 1976 |
+
yield _ocr_status_html(), "\n".join(log), 0 # stream: book start
|
| 1977 |
profile_path = PROFILES_DIR / f"{book_id}_page_profile.json"
|
| 1978 |
if not profile_path.exists():
|
| 1979 |
log.append(log_line(f"β {book_id}: no profile"))
|
|
|
|
| 2239 |
# Stream progress after each batch
|
| 2240 |
done_pages = min(start + batch_size, len(ocr_targets))
|
| 2241 |
log.append(log_line(f" β {book_id}: batch {start//batch_size + 1} complete ({done_pages}/{len(ocr_targets)} pages)"))
|
| 2242 |
+
yield _ocr_status_html(), "\n".join(log), 0
|
| 2243 |
|
| 2244 |
log.append(log_line(f" β {book_id}: batch OCR complete β building review queue"))
|
| 2245 |
+
yield _ocr_status_html(), "\n".join(log), 0 # stream: batch OCR done
|
| 2246 |
# Second pass: build page-level OCR output and review queue.
|
| 2247 |
for page_data in profile.get("pages", []):
|
| 2248 |
page_num = page_data["page_number"]
|
|
|
|
| 2371 |
df.loc[df["book_id"] == book_id, "status"] = "ocred"
|
| 2372 |
total_review = len(review_pages) + len(quarantine_pages)
|
| 2373 |
log.append(log_line(f"β {book_id}: {len(ocr_pages)} selected pp β review queue: {total_review} Β· queue write complete"))
|
| 2374 |
+
yield _ocr_status_html(), "\n".join(log), 0 # stream: book done
|
| 2375 |
|
| 2376 |
qdf_new = pd.DataFrame(queue_rows)
|
| 2377 |
if QUEUE_CSV.exists():
|
|
|
|
| 2395 |
QUEUE_CSV.unlink()
|
| 2396 |
|
| 2397 |
save_manifest_df(df)
|
| 2398 |
+
yield _ocr_status_html(), "\n".join(log), 1 # stream: final β triggers review load
|
| 2399 |
|
| 2400 |
|
| 2401 |
def _ocr_status_html() -> str:
|
|
|
|
| 2874 |
</div>""")
|
| 2875 |
|
| 2876 |
ocr_status = gr.HTML(_ocr_status_html())
|
| 2877 |
+
ocr_done = gr.State(0) # increments when OCR completes β triggers review load
|
| 2878 |
|
| 2879 |
gr.HTML("""<div class="ss-card">
|
| 2880 |
<div class="ss-card-title">Self-Improvement Loop</div>
|
|
|
|
| 2912 |
ocr_run_event = ocr_btn.click(
|
| 2913 |
run_ocr,
|
| 2914 |
inputs=[ocr_page_selection, ocr_page_exclusion, ocr_replace_queue],
|
| 2915 |
+
outputs=[ocr_status, ocr_log, ocr_done],
|
| 2916 |
show_progress="minimal",
|
| 2917 |
trigger_mode="once",
|
| 2918 |
)
|
|
|
|
| 3036 |
outputs=review_load_outputs
|
| 3037 |
)
|
| 3038 |
review_tab.select(load_review, outputs=review_load_outputs)
|
| 3039 |
+
ocr_done.change(load_review, outputs=review_load_outputs)
|
| 3040 |
+
|
|
|
|
|
|
|
|
|
|
| 3041 |
accept_btn.click(do_accept, inputs=[current_idx, final_text_box, reviewer_name, reason_code], outputs=action_outputs)
|
| 3042 |
edit_btn.click(do_accept, inputs=[current_idx, final_text_box, reviewer_name, reason_code], outputs=action_outputs)
|
| 3043 |
reject_btn.click(do_reject, inputs=[current_idx, final_text_box, reviewer_name, reason_code], outputs=action_outputs)
|